diff --git a/.circleci/config.yml b/.circleci/config.yml index 1045e671..39d8dc38 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,6 +98,31 @@ commands: - ~/.m2 key: v1-dependencies-{{ checksum "pom.xml" }} + install_asciidoctor: + steps: + - checkout + - run: + name: Install asciidoctor + command: | + sudo apt-get update + sudo apt install -y asciidoctor + + run_asciidoctor: + steps: + - run: + name: Run asciidoctor + command: | + cd docs + asciidoctor -a docinfo=shared index.adoc + + upload_docs_site: + steps: + - run: + name: scp html to justinsgotskinnylegs.com + command: | + cd docs + scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html + jobs: mvn_test: executor: localstack/default @@ -114,6 +139,13 @@ jobs: - mvn_verify - mvn_jar_deploy + publish_asciidoc: + executor: localstack/default + steps: + - install_asciidoctor + - run_asciidoctor + - upload_docs_site + workflows: test_only: jobs: @@ -121,7 +153,7 @@ workflows: context: [ qqq-maven-registry-credentials, build-qqq-sample-app ] filters: branches: - ignore: /dev/ + ignore: /(dev|integration.*)/ tags: ignore: /(version|snapshot)-.*/ @@ -131,7 +163,10 @@ workflows: context: [ qqq-maven-registry-credentials, build-qqq-sample-app ] filters: branches: - only: /dev/ + only: /(dev|integration.*)/ tags: only: /(version|snapshot)-.*/ - + - publish_asciidoc: + filters: + branches: + only: /dev/ diff --git a/docs/index.adoc b/docs/index.adoc index 1ab41a32..d777f0cc 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -26,6 +26,16 @@ include::metaData/Reports.adoc[leveloffset=+1] include::metaData/Icons.adoc[leveloffset=+1] include::metaData/PermissionRules.adoc[leveloffset=+1] +== Services + +include::misc/ScheduledJobs.adoc[leveloffset=+1] + +=== Web server (Javalin) +#todo# + +=== API server (OpenAPI) +#todo# + == Custom Application Code include::misc/QContext.adoc[leveloffset=+1] include::misc/QRecords.adoc[leveloffset=+1] @@ -63,4 +73,4 @@ include::implementations/TableSync.adoc[leveloffset=+1] // later... include::actions/RenderTemplateAction.adoc[leveloffset=+1] == QQQ Utility Classes -include::utilities/RecordLookupHelper.adoc[leveloffset=+1] \ No newline at end of file +include::utilities/RecordLookupHelper.adoc[leveloffset=+1] diff --git a/docs/metaData/Fields.adoc b/docs/metaData/Fields.adoc index 93d11dcd..eb20fa84 100644 --- a/docs/metaData/Fields.adoc +++ b/docs/metaData/Fields.adoc @@ -21,5 +21,99 @@ Used to set values in the `displayValues` map within a `QRecord`. * `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field. Values in this field should correspond to ids from the referenced Possible Value Source. * `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field. -Only applicable for fields with `type=STRING`. -* ` \ No newline at end of file +Only applicable for fields with `type=STRING`. Needs to be used with a `FieldBehavior` of type `ValueTooLongBehavior`. + +==== Field Behaviors +Additional behaviors can be attached to fields through the use of the `behaviors` attribute, +which is a `Set` of 0 or more instances of implementations of the `FieldBehavior` interface. +Note that in some cases, these instances may be `enum` constants, +but other times may be regular Objects. + +QQQ provides a set of common field behaviors. +Applications can also define their own field behaviors by implementing the `FieldBehavior` interface, +and attaching instances of their custom behavior classes to fields. + +===== ValueTooLongBehavior +Used on String fields. Requires the field to have a `maxLength` set. +Depending on the chosen instance of this enum, before a record is Inserted or Updated, +if the value in the field is longer than the `maxLength`, then one of the following actions can occur: + +* `TRUNCATE` - The value will be simply truncated to the `maxLength`. +* `TRUNCATE_ELLIPSIS` - The value will be truncated to 3 characters less than the `maxLength`, and three periods (an ellipsis) will be placed at the end. +* `ERROR` - An error will be reported, and the record will not be inserted or updated. +* `PASS_THROUGH` - Nothing will happen. This is the same as not having a `ValueTooLongBehavior` on the field. + +[source,java] +.Examples of using ValueTooLongBehavior +---- + new QFieldMetaData("sku", QFieldType.STRING) + .withMaxLength(40), + .withBehavior(ValueTooLongBehavior.ERROR), + + new QFieldMetaData("reason", QFieldType.STRING) + .withMaxLength(250), + .withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS), + +---- + +===== DynamicDefaultValueBehavior +Used to set a dynamic default value to a field when it is being inserted or updated. +For example, instead of having a hard-coded `defaultValue` specified in the field meta-data, +and instead of having to add, for example, a pre-insert custom action. + +* `CREATE_DATE` - On inserts, sets the field's value to the current time. +* `MODIFY_DATE` - On inserts and updates, sets the field's value to the current time. +* `USER_ID` - On inserts and updates, sets the field's value to the current user's id (but only if the value is currently null). + +_Note that the `QInstanceEnricher` will, by default, add the `CREATE_DATE` and `MODIFY_DATE` `DynamicDefaultValueBehavior` +options to any fields named `"createDate"` or `"modifyDate"`. +This behavior can be disabled by setting the `configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate` property +on the `QInstanceEnricher` instance used by the application to `false`._ + +[source,java] +.Examples of using DynamicDefaultValueBehavior +---- + new QFieldMetaData("createDate", QFieldType.DATE_TIME) + .withBehavior(DynamicDefaultValueBehavior.CREATE_DATE), + + new QFieldMetaData("modifyDate", QFieldType.DATE_TIME) + .withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE), + + new QFieldMetaData("createdByUserId", QFieldType.STRING) + .withBehavior(DynamicDefaultValueBehavior.USER_ID), +---- + +===== DateTimeDisplayValueBehavior +By default, in QQQ, fields of type `DATE_TIME` are stored in UTC, +and their values in a QRecord is a java `Instant` instance, which is always UTC. +However, frontends will prefer to display date-time values in the user's local Time Zone whenever possible. + +Using `DateTimeDisplayValueBehavior` allows a `DATE_TIME` field to be displayed in a different Time Zone. +An example use-case for this would be displaying airplane flight times, +where you would want a flight from California to New York to display Pacific Time for its departure time, +and Eastern Time for its arrival. + +An instance of `DateTimeDisplayValueBehavior` can be configured to either use a hard-coded time `ZoneId` +(for example, to always show users UTC, or a business's home-office time zone). +Or, it can be set up to get the time zone to use from another field in the table. + +[source,java] +.Examples of using DateTimeDisplayValueBehavior +---- +new QTableMetaData().withName("flights").withFields(List.of( + ... + new QFieldMetaData("departureTimeZoneId", QFieldType.STRING), + new QFieldMetaData("arrivaTimeZoneId", QFieldType.STRING), + + new QFieldMetaData("departureTime", QFieldType.DATE_TIME) + .withBehavior(new DateTimeDisplayValueBehavior() + .withZoneIdFromFieldName("departureTimeZoneId")), + + new QFieldMetaData("arrivalTime", QFieldType.DATE_TIME) + .withBehavior(new DateTimeDisplayValueBehavior() + .withZoneIdFromFieldName("arrivalTimeZoneId")) + + new QFieldMetaData("ticketSaleStartDateTime", QFieldType.DATE_TIME) + .withBehavior(new DateTimeDisplayValueBehavior() + .withDefaultZoneId("UTC")) +---- diff --git a/docs/metaData/Joins.adoc b/docs/metaData/Joins.adoc index 4ce09ea3..3d4adee7 100644 --- a/docs/metaData/Joins.adoc +++ b/docs/metaData/Joins.adoc @@ -2,16 +2,57 @@ == Joins include::../variables.adoc[] -#TODO# +A `QJoinMetaData` is a meta-data object that tells QQQ, essentially “it is possible for these 2 tables to join, here’s how to do it”. + +Joins can be used then, in an application, in a number of possible ways: + +* In a {link-table}, we can specify joins to be “exposed”, e.g., made available to users on a query screen +* Also in a Table, as part of an “Association”, which sets up one table as a “parent” of another, +such that you can store (and fetch) the child-records at the same time as the parent +** A common use-case here may be an order & lineItem table - +such that QQQ can generate an API uses to allow you to post an order and its lines in a single request, and they get stored all together +* In defining the security field (record lock) on a table, +sometimes, it isn’t a field directly on the table, but instead comes from a joined table (possibly even more than just 1 table away). +** For example, maybe a lineItem table, doesn't have a clientId, but needs secured by that field +- so its recordLock can specify a “joinNameChain” that describes how to get from lineItem to order.clientId +* The `QueryAction` can take (through its QueryInput object) zero or more QueryJoin objects, +which must make a reference (implicitly or explicitly) to a QJoinMetaData. +See the section on <> for more details. === QJoinMetaData Joins are defined in a QQQ Instance in a `*QJoinMetaData*` object. -#TODO# +In this object, we have the concept of a "leftTable" and a "rightTable". +There isn't generally anything special about which table is on the "left" and which is on the "right". +But the remaining pieces of the QJoinMetaData do all need to line-up with these sides. + +For example: + +* The Type (one-to-one, one-to-many, many-to-one) - where the leftTable comes first, and rightTable comes second +(e.g., a one-to-many means 1-row in leftTable has many-rows in rightTable associated with it) +* In a JoinOn object, the 1st field name given is from the leftTable; +the second fieldName from the rightTable. *QJoinMetaData Properties:* -* `name` - *String, Required* - Unique name for the join within the QQQ Instance. #todo infererences or conventions?# - -#TODO# +* `name` - *String, Required* - Unique name for the join within the QQQ Instance. +** One convention is to name joins based on (leftTable + "Join" + rightTable). +** If you do not wish to define join names yourself, the method `withInferredName()` +can be called (which defers to +`public static String makeInferredJoinName(String leftTable, String rightTable)`), +to create a name for the join following the (leftTable + "Join" + rightTable) convention. +* `leftTable` - *String, Required* - Name of a {link-table} in the {link-instance}. +* `rightTable` - *String, Required* - Name of a {link-table} in the {link-instance}. +* `type` - *enum, Required* - cardinality between the two tables in the join. +** e.g., `ONE_TO_ONE`, `ONE_TO_MANY` (indicating 1 record in the left table may join +to many records in the right table), or `MANY_TO_ONE` (vice-versa). +** Note that there is no MANY_TO_MANY option, as a many-to-many is built as multiple QJoinMetaData's +going through the intermediary (intersection) table. +* `joinOns` - *List, Required* - fields used to join the tables. +Note: In the 2-arg JoinOn constructor, the leftTable's field comes first. +Alternatively, the no-arg constructor can be used along with `.withLeftField().withRightField()` +* `orderBys` - *List* - Optional list of order-by objects, +used in some framework-generated queries using the join. +The field names are assumed to come from the rightTable. +#TODO# what else do we need here? diff --git a/docs/metaData/Processes.adoc b/docs/metaData/Processes.adoc index 253f6318..2232b3f7 100644 --- a/docs/metaData/Processes.adoc +++ b/docs/metaData/Processes.adoc @@ -16,6 +16,7 @@ Processes are defined in a QQQ Instance in a `*QProcessMetaData*` object. In addition to directly building a `QProcessMetaData` object setting its properties directly, there are a few common process patterns that provide *Builder* objects for ease-of-use. See StreamedETLWithFrontendProcess below for a common example +[#_QProcessMetaData_Properties] *QProcessMetaData Properties:* * `name` - *String, Required* - Unique name for the process within the QQQ Instance. @@ -30,12 +31,13 @@ See below for details. * `permissionRules` - *QPermissionRules object* - define the permission/access rules for the process. See {link-permissionRules} for details. * `steps` and `stepList` - *Map of String → <>* and *List of QStepMetaData* - Defines the <> and <> that makes up the process. -** `stepList` is the list of steps in the order that they will by default be executed. -** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time. +** `stepList` is the list of steps in the order that they will be executed +(that is to say - this is the _default_ order of execution - but it can be customized - see <<_custom_process_flow>> for details). +** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time (e.g., using <<_custom_process_flow>>). ** A process's steps are normally defined in one of two was: *** 1) by a single call to `.withStepList(List)`, which internally adds each step into the `steps` map. *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map. -** If a process also needs optional steps, they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map. +** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map. * `schedule` - *<>* - set up the process to run automatically on the specified schedule. See below for details. * `minInputRecords` - *Integer* - #not used...# @@ -214,3 +216,112 @@ But for some cases, doing page-level transactions can reduce long-transactions a * `withFields(List fieldList)` - Adds additional input fields to the preview step of the process. * `withBasepullConfiguration(BasepullConfiguration basepullConfiguration)` - Add a <> to the process. * `withSchedule(QScheduleMetaData schedule)` - Add a <> to the process. + +[#_custom_process_flow] +==== Custom Process Flow +As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process +will execute each of its steps in-order, as defined in the `stepList` property. +However, a Backend Step can customize this flow #todo - write more clearly here... + +There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow: + +* `RunBackendStepOutput.setOverrideLastStepName(String stepName)` +** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next. +However, if a step sets the `OverrideLastStepName` property in its output object, +then the step named in that property becomes the effective "last" step, +thus determining which step comes next. + +* `RunBackendStepOutput.updateStepList(List stepNameList)` +** Calling this method changes the process's runtime definition of steps to be executed. +Thus allowing a completely custom flow. +It should be noted, that the "last" step name (as tracked by QQQ within `RunProcessAction`) +does need to be found in the new `stepNameList` - otherwise, the framework will not know where you were, +for figuring out where to go next. + +[source,java] +.Example of a defining process that can use a flexible flow: +---- +// for a case like this, it would be recommended to define all step names in constants: +public final static String STEP_START = "start"; +public final static String STEP_A = "a"; +public final static String STEP_B = "b"; +public final static String STEP_C = "c"; +public final static String STEP_1 = "1"; +public final static String STEP_2 = "2"; +public final static String STEP_3 = "3"; +public final static String STEP_END = "end"; + +// also, to define the possible flows (lists of steps) in constants as well: +public final static List LETTERS_STEP_LIST = List.of( + STEP_START, STEP_A, STEP_B, STEP_C, STEP_END); + +public final static List NUMBERS_STEP_LIST = List.of( + STEP_START, STEP_1, STEP_2, STEP_3, STEP_END); + +// when we define the process's meta-data, we only give a "skeleton" stepList - +// we must at least have our starting step, and we may want at least one frontend step +// for the UI to show some placeholder(s): +QProcessMetaData process = new QProcessMetaData() + .withName(PROCESS_NAME) + .withStepList(List.of( + new QBackendStepMetaData().withName(STEP_START) + .withCode(new QCodeReference(/*...*/)), + new QFrontendStepMetaData() + .withName(STEP_END) + )); + +// the additional steps get added via `addOptionalStep`, which only puts them in +// the process's stepMap, not its stepList! +process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_A)); +process.addOptionalStep(new QBackendStepMetaData().withName(STEP_B) + .withCode(new QCodeReference(/*...*/))); +process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_C)); + +process.addOptionalStep(new QBackendStepMetaData().withName(STEP_1) + .withCode(new QCodeReference(/*...*/))); +process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_2)); +process.addOptionalStep(new QBackendStepMetaData().withName(STEP_3) + .withCode(new QCodeReference(/*...*/))); + +---- + +[source,java] +.Example of a process backend step adjusting the process's runtime flow: +---- +/*************************************************************************** +** look at the value named "which". if it's "letters", then make the process +** go through the stepList consisting of letters; else, update the step list +** to be the "numbers" steps. +** +** Also - if the "skipSomeSteps" value is give as true, then set the +** overrideLastStepName to skip again (in the letters case, skip past A, B +** and C; in the numbers case, skip past 1 and 2). +***************************************************************************/ +public static class StartStep implements BackendStep +{ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + Boolean skipSomeSteps = runBackendStepInput.getValueBoolean("skipSomeSteps"); + + if(runBackendStepInput.getValueString("which").equals("letters")) + { + runBackendStepOutput.updateStepList(LETTERS_STEP_LIST); + if(BooleanUtils.isTrue(skipSomeSteps)) + { + runBackendStepOutput.setOverrideLastStepName(STEP_C); + } + } + else + { + runBackendStepOutput.updateStepList(NUMBERS_STEP_LIST); + if(BooleanUtils.isTrue(skipSomeSteps)) + { + runBackendStepOutput.setOverrideLastStepName(STEP_2); + } + } + } +} +---- + + diff --git a/docs/misc/ScheduledJobs.adoc b/docs/misc/ScheduledJobs.adoc new file mode 100644 index 00000000..b4a88a61 --- /dev/null +++ b/docs/misc/ScheduledJobs.adoc @@ -0,0 +1,141 @@ +== Schedulers and Scheduled Jobs +include::../variables.adoc[] + +QQQ has the ability to automatically run various types of jobs on schedules, +either defined in your instance's meta-data, +or optionally via data in your application, in a `scheduledJob` table. + +=== Schedulers and QSchedulerMetaData +2 types of schedulers are included in QQQ by default (though an application can define its own schedulers): + +* `SimpleScheduler` - is (as its name suggests) a simple class which uses java's `ScheduledExecutorService` +to run jobs on repeating intervals. +** Cannot run cron schedules - only repeating intervals. +** If multiple servers are running, each will potentially run the same job concurrently +** Has no configurations, e.g., to limit the number of threads. + +* `QuartzScheduler` - uses the 3rd party https://www.quartz-scheduler.org/[Quartz Scheduler] library to provide +a much more capable, though admittedly more complex, scheduling solution. +** Can run both cron schedules and repeating intervals. +** By default, will not allow concurrent executions of the same job. +** Supports multiple configurations, e.g., to limit the number of threads. + +An application can define its own scheduler by providing a class which implements the `QSchedulerInterface`. + +A `QInstance` can work with 0 or more schedulers, as defined by adding `QSchedulerMetaData` objects +to the instance. + +This meta-data class is `abstract`, and is extended by the 2 built-in schedulers +(e.g., `SimpleSchedulerMetaData` and `QuartzSchedulerMetaData`). As such, +these concrete subclasses are what you need to instantiate and add to your instance. + +To configure a QuartzScheduler, you can add a `Properties` object to the `QuartzSchedulerMetaData` object. +See https://www.quartz-scheduler.org/documentation/[Quartz's documentation] for available configuration properties. + +[source,java] +.Defining SchedulerMetaData +---- +qInstance.addScheduler(new SimpleSchedulerMetaData().withName("mySimpleScheduler")); + +qInstance.addScheduler(new QuartzSchedulerMetaData() + .withName("myQuartzScheduler") + .withProperties(myQuartzProperties); +---- + +=== SchedulableTypes +The types of jobs which can be scheduled in a QQQ application are defined in the `QInstance` by +instances of the `SchedulableType` meta-data class. +These objects contain a name, along with a `QCodeReference` to the `runner`, +which must be a class that implements the `SchedulableRunner` interface. + +By default, (in the `QInstanceEnricher`), QQQ will make 3 `SchedulableType` options available: + +* `PROCESS` - Any Process defined in the `QInstance` can be scheduled. +* `QUEUE_PROCESSOR` - A Queue defined in the `QInstance`, which requires polling (e.g., SQS), can be scheduled. +* `TABLE_AUTOMATIONS` - A Table in the `QInstance`, with `AutomationDetails` referring to an +AutomationProvider which requires polling, can be scheduled. + +If an application only wants to use a subset of these `SchedulableType` options, +or to add custom `SchedulableType` options, +the `QInstance` will need to have 1 or more `SchedulableType` objects in it before the `QInstanceEnricher` runs. + +=== User-defined Scheduled Jobs +To allow users to schedule jobs (rather than using programmer-defined schedules (in meta-data)), +you can add a set of tables to your `QInstance`, using the `ScheduledJobsMetaDataProvider` class: + +[source,java] +.Adding the ScheduledJob tables and related meta-data to a QInstance +---- +new ScheduledJobsMetaDataProvider().defineAll( + qInstance, backendName, table -> tableEnricher(table)); +---- + +This meta-data provider adds a "scheduledJob" and "scheduledJobParameter" table, along with +some PossibleValueSources. +These tables include post-action customizers, which manage (re-, un-) scheduling jobs based on +changes made to records in this these tables. + +Also, when `QScheduleManager` is started, it will query these tables,and will schedule jobs as defined therein. + +_You can use a mix of user-defined and meta-data defined scheduled jobs in your instance. +However, if a ScheduledJob record references a process, queue, or table automation with a +meta-data defined schedule, the ScheduledJob will NOT be started by ScheduleManager -- +rather, the meta-data definition will "win"._ + +[source,sql] +.Example of inserting scheduled jobs records directly into an SQL backend +---- +-- A process: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('myProcess', 'QuartzScheduler', null, null, 300, 'PROCESS', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'myProcess'), 'processName', 'myProcess'); + +-- A table's insert & update automations: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('myTable.PENDING_INSERT_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1), + ('myTable.PENDING_UPDATE_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'tableName', 'myTable'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'automationStatus', 'PENDING_INSERT_AUTOMATIONS'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'tableName', 'myTable'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'automationStatus', 'PENDING_UPDATE_AUTOMATIONS'); + +-- A queue processor: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('mySqsQueue', 'QuartzScheduler', null, null, 60, 'QUEUE_PROCESSOR', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'mySqsQueue'), 'queueName', 'mySqsQueue'); +---- + +=== Running Scheduled Jobs +In a server running QQQ, if you wish to start running scheduled jobs, you need to initialize +the `QScheduleManger` singleton class, then call its `start()` method. + +Note that internally, this class will check for a system property of `qqq.scheduleManager.enabled` +or an environment variable of `QQQ_SCHEDULE_MANAGER_ENABLED`, and if the first value found is +`"false"`, then the scheduler will not actually run its jobs (but, in the case of the `QuartzSchdeuler`, +it will be available for managing scheduled jobs). + +The method `QScheduleManager.initInstance` requires 2 parameters: Your `QInstance`, and a +`Supplier` lambda, which returns the session that will be used for scheduled jobs when they +are executed. + +[source,java] +.Starting the Schedule Manager service +---- +QScheduleManager.initInstance(qInstance, () -> systemUserSession).start(); +---- + +=== Examples +[source,java] +.Attach a schedule in meta-data to a Process +---- +QProcessMetaData myProcess = new QProcessMetaData() + // ... + .withSchedule(new QScheduleMetaData() + .withSchedulerName("myScheduler") + .withDescription("Run myProcess every five minutes") + .withRepeatSeconds(300)) +---- + diff --git a/pom.xml b/pom.xml index 79723366..a96d408b 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,7 @@ pom + qqq-bom qqq-backend-core qqq-backend-module-api qqq-backend-module-filesystem @@ -54,7 +55,7 @@ true true true - 0.80 + 0.75 0.95 none @@ -149,7 +150,7 @@ com.puppycrawl.tools checkstyle - 9.0 + 10.16.0 @@ -245,8 +246,7 @@ echo " See also target/site/jacoco/index.html" echo " and https://www.jacoco.org/jacoco/trunk/doc/counters.html" echo "------------------------------------------------------------" -which xpath > /dev/null 2>&1 -if [ "$?" == "0" ]; then +if which xpath > /dev/null 2>&1; then echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' @@ -255,8 +255,7 @@ else echo "xpath is not installed. Jacoco coverage summary will not be produced here..."; fi -which xpath > /dev/null 2>&1 -if [ "$?" == "0" ]; then +if which html2text > /dev/null 2>&1; then echo "Untested classes, per Jacoco:" echo "-----------------------------" for i in target/site/jacoco/*/index.html; do diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index 58191fa6..60561f25 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -102,6 +102,16 @@ fastexcel 0.12.15 + + org.apache.poi + poi + 5.2.5 + + + org.apache.poi + poi-ooxml + 5.2.5 + com.auth0 auth0 @@ -163,6 +173,38 @@ 1.12.321 + + com.amazonaws + aws-java-sdk-ses + 1.12.705 + + + + cloud.localstack + localstack-utils + 0.2.20 + test + + + + org.quartz-scheduler + quartz + 2.3.2 + + + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.23.0 + + + + com.sun.mail + jakarta.mail + 2.0.1 + + org.apache.maven.plugins diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index e4871048..875097fa 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -159,7 +159,7 @@ public class AsyncJobManager private T runAsyncJob(String jobName, AsyncJob asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus) { String originalThreadName = Thread.currentThread().getName(); - Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8)); + Thread.currentThread().setName("Job:" + jobName); try { LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid()); @@ -169,17 +169,24 @@ public class AsyncJobManager LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid()); return (result); } - catch(Exception e) + catch(Throwable t) { asyncJobStatus.setState(AsyncJobState.ERROR); - asyncJobStatus.setCaughtException(e); + if(t instanceof Exception e) + { + asyncJobStatus.setCaughtException(e); + } + else + { + asyncJobStatus.setCaughtException(new QException("Caught throwable", t)); + } getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); ////////////////////////////////////////////////////// // if user facing, just log an info, warn otherwise // ////////////////////////////////////////////////////// - LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid())); - throw (new CompletionException(e)); + LOG.log((t instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", t, logPair("jobId", uuidAndTypeStateKey.getUuid())); + throw (new CompletionException(t)); } finally { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java index 3ec54516..c5cf95f3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java @@ -31,6 +31,7 @@ import java.io.Serializable; *******************************************************************************/ public class AsyncJobStatus implements Serializable { + private String jobName; private AsyncJobState state; private String message; private Integer current; @@ -187,4 +188,36 @@ public class AsyncJobStatus implements Serializable { this.cancelRequested = cancelRequested; } + + + + /******************************************************************************* + ** Getter for jobName + *******************************************************************************/ + public String getJobName() + { + return (this.jobName); + } + + + + /******************************************************************************* + ** Setter for jobName + *******************************************************************************/ + public void setJobName(String jobName) + { + this.jobName = jobName; + } + + + + /******************************************************************************* + ** Fluent setter for jobName + *******************************************************************************/ + public AsyncJobStatus withJobName(String jobName) + { + this.jobName = jobName; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java new file mode 100644 index 00000000..2373dc79 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.async; + + +import java.util.UUID; + + +/******************************************************************************* + ** subclass designed to be used when we want there to be an instance (so code + ** doesn't have to all be null-tolerant), but there's no one who will ever be + ** reading the status data, so we don't need to store the object in a + ** state provider. + *******************************************************************************/ +public class NonPersistedAsyncJobCallback extends AsyncJobCallback +{ + private final AsyncJobStatus asyncJobStatus; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public NonPersistedAsyncJobCallback(UUID jobUUID, AsyncJobStatus asyncJobStatus) + { + super(jobUUID, asyncJobStatus); + this.asyncJobStatus = asyncJobStatus; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected void storeUpdatedStatus() + { + /////////////////////////////////////////////////////////////////////////////////////// + // noop - cf. base class, which writes to persistence here (our point is, we do not) // + /////////////////////////////////////////////////////////////////////////////////////// + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java index 3ab02ce4..a1183ba4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java @@ -149,7 +149,7 @@ public class DMLAuditAction extends AbstractQActionFunction sortedFieldNames = table.getFields().keySet().stream() - .sorted(Comparator.comparing(fieldName -> table.getFields().get(fieldName).getLabel())) + .sorted(Comparator.comparing(fieldName -> Objects.requireNonNullElse(table.getFields().get(fieldName).getLabel(), fieldName))) .toList(); QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField()); @@ -303,7 +303,7 @@ public class DMLAuditAction extends AbstractQActionFunction /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") public String getInsertOrUpdate() { return switch(this) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java index 2113facc..ba1118a4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java @@ -132,6 +132,11 @@ public class PollingAutomationPerTableRunner implements Runnable *******************************************************************************/ String tableName(); + /******************************************************************************* + ** + *******************************************************************************/ + QTableAutomationDetails tableAutomationDetails(); + /******************************************************************************* ** *******************************************************************************/ @@ -143,7 +148,7 @@ public class PollingAutomationPerTableRunner implements Runnable /******************************************************************************* ** Wrapper for a pair of (tableName, automationStatus) *******************************************************************************/ - public record TableActions(String tableName, AutomationStatus status) implements TableActionsInterface + public record TableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status) implements TableActionsInterface { /******************************************************************************* ** @@ -159,7 +164,7 @@ public class PollingAutomationPerTableRunner implements Runnable ** extended version of TableAction, for sharding use-case - adds the shard ** details. *******************************************************************************/ - public record ShardedTableActions(String tableName, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface + public record ShardedTableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface { /******************************************************************************* ** @@ -198,8 +203,8 @@ public class PollingAutomationPerTableRunner implements Runnable { Serializable shardId = record.getValue(automationDetails.getShardIdFieldName()); String label = record.getValueString(automationDetails.getShardLabelFieldName()); - tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); - tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); + tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); + tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); } } catch(Exception e) @@ -209,11 +214,11 @@ public class PollingAutomationPerTableRunner implements Runnable } else { - /////////////////////////////////////////////////////////////////// - // for non-sharded, we just need tabler name & automation status // - /////////////////////////////////////////////////////////////////// - tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS)); - tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + ////////////////////////////////////////////////////////////////// + // for non-sharded, we just need table name & automation status // + ////////////////////////////////////////////////////////////////// + tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); } } } @@ -521,7 +526,7 @@ public class PollingAutomationPerTableRunner implements Runnable // note - this method - will re-query the objects, so we should have confidence that their data is fresh... // ////////////////////////////////////////////////////////////////////////////////////////////////////////////// List matchingQRecords = getRecordsMatchingActionFilter(table, records, action); - LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); + LOG.debug("Of the [" + records.size() + "] records that were pending automations, [" + matchingQRecords.size() + "] of them match the filter on the action:" + action); if(CollectionUtils.nullSafeHasContents(matchingQRecords)) { LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action); @@ -596,7 +601,7 @@ public class PollingAutomationPerTableRunner implements Runnable /******************************************************************************* ** Finally, actually run action code against a list of known matching records. - ** todo not commit - move to somewhere genericer + ** *******************************************************************************/ public static void applyActionToMatchingRecords(QTableMetaData table, List records, TableAutomationAction action) throws Exception { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java index edf9a9cf..52864385 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java @@ -47,12 +47,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** records that the delete action marked in error - the user might want to do ** something special with them (idk, try some other way to delete them?) *******************************************************************************/ -public abstract class AbstractPostDeleteCustomizer +public abstract class AbstractPostDeleteCustomizer implements TableCustomizerInterface { protected DeleteInput deleteInput; + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + this.deleteInput = deleteInput; + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java index c7e2bfc6..100fe267 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java @@ -42,12 +42,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** ** Note that the full insertInput is available as a field in this class. *******************************************************************************/ -public abstract class AbstractPostInsertCustomizer +public abstract class AbstractPostInsertCustomizer implements TableCustomizerInterface { protected InsertInput insertInput; + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + this.insertInput = insertInput; + return (apply(records)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java index d1beaa4c..669aa06b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java @@ -23,16 +23,29 @@ package com.kingsrook.qqq.backend.core.actions.customizers; import java.util.List; -import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* ** *******************************************************************************/ -public abstract class AbstractPostQueryCustomizer +public abstract class AbstractPostQueryCustomizer implements TableCustomizerInterface { - protected AbstractTableActionInput input; + protected QueryOrGetInputInterface input; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface queryInput, List records) throws QException + { + input = queryInput; + return apply(records); + } @@ -47,7 +60,7 @@ public abstract class AbstractPostQueryCustomizer ** Getter for input ** *******************************************************************************/ - public AbstractTableActionInput getInput() + public QueryOrGetInputInterface getInput() { return (input); } @@ -58,7 +71,7 @@ public abstract class AbstractPostQueryCustomizer ** Setter for input ** *******************************************************************************/ - public void setInput(AbstractTableActionInput input) + public void setInput(QueryOrGetInputInterface input) { this.input = input; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java index b0d55b35..53e00583 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -48,7 +49,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** available (if the backend supports it) - both as a list (`getOldRecordList`) ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`). *******************************************************************************/ -public abstract class AbstractPostUpdateCustomizer +public abstract class AbstractPostUpdateCustomizer implements TableCustomizerInterface { protected UpdateInput updateInput; protected List oldRecordList; @@ -57,6 +58,19 @@ public abstract class AbstractPostUpdateCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + this.updateInput = updateInput; + this.oldRecordList = oldRecordList.orElse(null); + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java index 4b848a14..80460a86 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java @@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** Note that the full deleteInput is available as a field in this class. ** *******************************************************************************/ -public abstract class AbstractPreDeleteCustomizer +public abstract class AbstractPreDeleteCustomizer implements TableCustomizerInterface { protected DeleteInput deleteInput; @@ -58,6 +58,19 @@ public abstract class AbstractPreDeleteCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preDelete(DeleteInput deleteInput, List records, boolean isPreview) throws QException + { + this.deleteInput = deleteInput; + this.isPreview = isPreview; + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java index 196ea4b8..c0706a5c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java @@ -47,7 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** ** Note that the full insertInput is available as a field in this class. *******************************************************************************/ -public abstract class AbstractPreInsertCustomizer +public abstract class AbstractPreInsertCustomizer implements TableCustomizerInterface { protected InsertInput insertInput; @@ -70,6 +70,30 @@ public abstract class AbstractPreInsertCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + this.insertInput = insertInput; + this.isPreview = isPreview; + return (apply(records)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return getWhenToRun(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java index b8a95ed2..701ce30c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -53,7 +54,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** available (if the backend supports it) - both as a list (`getOldRecordList`) ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`). *******************************************************************************/ -public abstract class AbstractPreUpdateCustomizer +public abstract class AbstractPreUpdateCustomizer implements TableCustomizerInterface { protected UpdateInput updateInput; protected List oldRecordList; @@ -63,6 +64,20 @@ public abstract class AbstractPreUpdateCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + this.updateInput = updateInput; + this.isPreview = isPreview; + this.oldRecordList = oldRecordList.orElse(null); + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index 05acf79a..b4d46ca7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.actions.customizers; +import java.lang.reflect.Constructor; import java.util.Optional; import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; @@ -34,42 +35,35 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** Utility to load code for running QQQ customizers. + ** + ** TODO - redo all to go through method that memoizes class & constructor + ** lookup. That memoziation causes 1,000,000 such calls to go from ~500ms + ** to ~100ms. *******************************************************************************/ public class QCodeLoader { private static final QLogger LOG = QLogger.getLogger(QCodeLoader.class); - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Optional> getTableCustomizerFunction(QTableMetaData table, String customizerName) - { - Optional codeReference = table.getCustomizer(customizerName); - if(codeReference.isPresent()) - { - return (Optional.ofNullable(QCodeLoader.getFunction(codeReference.get()))); - } - return (Optional.empty()); - } + private static Memoization> constructorMemoization = new Memoization<>(); /******************************************************************************* ** *******************************************************************************/ - public static Optional getTableCustomizer(Class expectedClass, QTableMetaData table, String customizerName) + public static Optional getTableCustomizer(QTableMetaData table, String customizerName) { Optional codeReference = table.getCustomizer(customizerName); if(codeReference.isPresent()) { - return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get()))); + return (Optional.ofNullable(QCodeLoader.getAdHoc(TableCustomizerInterface.class, codeReference.get()))); } return (Optional.empty()); } @@ -102,7 +96,7 @@ public class QCodeLoader } catch(Exception e) { - LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e); + LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference)); ////////////////////////////////////////////////////////////////////////////////////////////////////////// // return null here - under the assumption that during normal run-time operations, we'll never hit here // @@ -141,7 +135,7 @@ public class QCodeLoader } catch(Exception e) { - LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e); + LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference)); ////////////////////////////////////////////////////////////////////////////////////////////////////////// // return null here - under the assumption that during normal run-time operations, we'll never hit here // @@ -175,12 +169,25 @@ public class QCodeLoader try { - Class customizerClass = Class.forName(codeReference.getName()); - return ((T) customizerClass.getConstructor().newInstance()); + Optional> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction, Exception>) s -> + { + Class customizerClass = Class.forName(codeReference.getName()); + return customizerClass.getConstructor(); + }); + + if(constructor.isPresent()) + { + return ((T) constructor.get().newInstance()); + } + else + { + LOG.error("Could not get constructor for code reference", logPair("codeReference", codeReference)); + return (null); + } } catch(Exception e) { - LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e); + LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference)); ////////////////////////////////////////////////////////////////////////////////////////////////////////// // return null here - under the assumption that during normal run-time operations, we'll never hit here // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java index ff8c2d82..afb54a71 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java @@ -50,10 +50,7 @@ public interface RecordCustomizerUtilityInterface /******************************************************************************* ** Container for an old value and a new value. *******************************************************************************/ - @SuppressWarnings("checkstyle:MethodName") - record Change(Serializable oldValue, Serializable newValue) - { - } + record Change(Serializable oldValue, Serializable newValue) {} /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java new file mode 100644 index 00000000..3a7cfa41 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java @@ -0,0 +1,202 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Common interface used by all (core) TableCustomizer types (e.g., post-query, + ** and {pre,post}-{insert,update,delete}. + ** + ** Note that the abstract-base classes for each action still exist, though have + ** been back-ported to be implementors of this interface. The action classes + ** will now expect this type, and call this type's methods. + ** + *******************************************************************************/ +public interface TableCustomizerInterface +{ + QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class); + + + /******************************************************************************* + ** custom actions to run after a query (or get!) takes place. + ** + *******************************************************************************/ + default List postQuery(QueryOrGetInputInterface queryInput, List records) throws QException + { + LOG.info("A default implementation of postQuery is running... Probably not expected!", logPair("tableName", queryInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions before an insert takes place. + ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (the inputs to + ** the insert action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly manipulating values (`setValue`) + ** - possibly throwing an exception - if you really don't want the insert operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class. + *******************************************************************************/ + default List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + LOG.info("A default implementation of preInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); + } + + + /******************************************************************************* + ** custom actions after an insert takes place. + ** + ** General implementation would be, to iterate over the records (the outputs of + ** the insert action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly throwing an exception - though doing so won't stop the update, and instead + ** will just set a warning on all of the updated records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back to the caller. + *******************************************************************************/ + default List postInsert(InsertInput insertInput, List records) throws QException + { + LOG.info("A default implementation of postInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions before an update takes place. + ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (the inputs to + ** the update action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly manipulating values (`setValue`) + ** - possibly throwing an exception - if you really don't want the update operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class. + ** + ** Note, "old records" (e.g., with values freshly fetched from the backend) will be + ** available (if the backend supports it) + *******************************************************************************/ + default List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + LOG.info("A default implementation of preUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions after an update takes place. + ** + ** General implementation would be, to iterate over the records (the outputs of + ** the update action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records? + ** - possibly throwing an exception - though doing so won't stop the update, and instead + ** will just set a warning on all of the updated records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back to the caller. + ** + ** Note, "old records" (e.g., with values freshly fetched from the backend) will be + ** available (if the backend supports it). + *******************************************************************************/ + default List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + LOG.info("A default implementation of postUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** Custom actions before a delete takes place. + ** + ** It's important for implementations to be aware of the isPreview param, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (which the DeleteAction + ** would look up based on the inputs to the delete action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly throwing an exception - if you really don't want the delete operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) - this is how errors + ** and warnings are propagated to the DeleteAction. Note that any records with + ** an error will NOT proceed to the backend's delete interface - but those with + ** warnings will. + *******************************************************************************/ + default List preDelete(DeleteInput deleteInput, List records, boolean isPreview) throws QException + { + LOG.info("A default implementation of preDelete is running... Probably not expected!", logPair("tableName", deleteInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** Custom actions after a delete takes place. + ** + ** General implementation would be, to iterate over the records (ones which didn't + ** have a delete error), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records? + ** - possibly throwing an exception - though doing so won't stop the delete, and instead + ** will just set a warning on all of the deleted records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back + ** to the caller - this is how errors and warnings are propagated . + *******************************************************************************/ + default List postDelete(DeleteInput deleteInput, List records) throws QException + { + LOG.info("A default implementation of postDelete is running... Probably not expected!", logPair("tableName", deleteInput.getTableName())); + return (records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java index 2c753b56..4c4d0f8d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java @@ -29,13 +29,13 @@ package com.kingsrook.qqq.backend.core.actions.customizers; *******************************************************************************/ public enum TableCustomizers { - POST_QUERY_RECORD("postQueryRecord", AbstractPostQueryCustomizer.class), - PRE_INSERT_RECORD("preInsertRecord", AbstractPreInsertCustomizer.class), - POST_INSERT_RECORD("postInsertRecord", AbstractPostInsertCustomizer.class), - PRE_UPDATE_RECORD("preUpdateRecord", AbstractPreUpdateCustomizer.class), - POST_UPDATE_RECORD("postUpdateRecord", AbstractPostUpdateCustomizer.class), - PRE_DELETE_RECORD("preDeleteRecord", AbstractPreDeleteCustomizer.class), - POST_DELETE_RECORD("postDeleteRecord", AbstractPostDeleteCustomizer.class); + POST_QUERY_RECORD("postQueryRecord", TableCustomizerInterface.class), + PRE_INSERT_RECORD("preInsertRecord", TableCustomizerInterface.class), + POST_INSERT_RECORD("postInsertRecord", TableCustomizerInterface.class), + PRE_UPDATE_RECORD("preUpdateRecord", TableCustomizerInterface.class), + POST_UPDATE_RECORD("postUpdateRecord", TableCustomizerInterface.class), + PRE_DELETE_RECORD("preDeleteRecord", TableCustomizerInterface.class), + POST_DELETE_RECORD("postDeleteRecord", TableCustomizerInterface.class); private final String role; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java index 26481e78..2563079f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java @@ -161,7 +161,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer public static String linkTableCreateWithDefaultValues(RenderWidgetInput input, String tableName, Map defaultValues) throws QException { String tablePath = QContext.getQInstance().getTablePath(tableName); - return (tablePath + "/create?defaultValues=" + URLEncoder.encode(JsonUtils.toJson(defaultValues), Charset.defaultCharset())); + return (tablePath + "/create#defaultValues=" + URLEncoder.encode(JsonUtils.toJson(defaultValues), Charset.defaultCharset())); } @@ -183,7 +183,6 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java index 2afb5f69..2bb15cfc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -72,80 +73,104 @@ public abstract class AbstractWidgetRenderer *******************************************************************************/ protected boolean setupDropdowns(RenderWidgetInput input, QWidgetMetaData metaData, QWidgetData widgetData) throws QException { - List>> pvsData = new ArrayList<>(); - List pvsLabels = new ArrayList<>(); - List pvsNames = new ArrayList<>(); + List>> dataList = new ArrayList<>(); + List labelList = new ArrayList<>(); + List nameList = new ArrayList<>(); List missingRequiredSelections = new ArrayList<>(); for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns())) { - String possibleValueSourceName = dropdownData.getPossibleValueSourceName(); - QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName); - - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // this looks complicated, but is just look for a label in the dropdown data and if found use it, // - // otherwise look for label in PVS and if found use that, otherwise just use the PVS name // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName); - pvsLabels.add(pvsLabel); - pvsNames.add(possibleValueSourceName); - - SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput(); - pvsInput.setPossibleValueSourceName(possibleValueSourceName); - - if(dropdownData.getForeignKeyFieldName() != null) + if(WidgetDropdownType.DATE_PICKER.equals(dropdownData.getType())) { - //////////////////////////////////////// - // look for an id in the query params // - //////////////////////////////////////// - Integer id = null; - if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id"))) + String name = dropdownData.getName(); + nameList.add(name); + labelList.add(dropdownData.getLabel()); + dataList.add(new ArrayList<>()); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // sure that something has been selected, and if not, display a message that a selection needs made // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(dropdownData.getIsRequired()) { - id = Integer.parseInt(input.getQueryParams().get("id")); - } - if(id != null) - { - pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria( - new QFilterCriteria( - dropdownData.getForeignKeyFieldName(), - QCriteriaOperator.EQUALS, - id))); + if(!input.getQueryParams().containsKey(name) || !StringUtils.hasContent(input.getQueryParams().get(name))) + { + missingRequiredSelections.add(dropdownData.getLabel()); + } } } - - SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput); - - List> dropdownOptionList = new ArrayList<>(); - pvsData.add(dropdownOptionList); - - ////////////////////////////////////////// - // sort results, dedupe, and add to map // - ////////////////////////////////////////// - Set exists = new HashSet<>(); - output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel())); - for(QPossibleValue possibleValue : output.getResults()) + else { - dropdownOptionList.add(MapBuilder.of( - "id", String.valueOf(possibleValue.getId()), - "label", possibleValue.getLabel() - )); - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // because we know the dropdowns and what the field names will be when something is selected, we can make // - // sure that something has been selected, and if not, display a message that a selection needs made // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(dropdownData.getIsRequired()) - { - if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName))) + String possibleValueSourceName = dropdownData.getPossibleValueSourceName(); + if(possibleValueSourceName != null) { - missingRequiredSelections.add(pvsLabel); + QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // this looks complicated, but is just look for a label in the dropdown data and if found use it, // + // otherwise look for label in PVS and if found use that, otherwise just use the PVS name // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName); + labelList.add(pvsLabel); + nameList.add(possibleValueSourceName); + + SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput(); + pvsInput.setPossibleValueSourceName(possibleValueSourceName); + + if(dropdownData.getForeignKeyFieldName() != null) + { + //////////////////////////////////////// + // look for an id in the query params // + //////////////////////////////////////// + Integer id = null; + if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id"))) + { + id = Integer.parseInt(input.getQueryParams().get("id")); + } + if(id != null) + { + pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria( + new QFilterCriteria( + dropdownData.getForeignKeyFieldName(), + QCriteriaOperator.EQUALS, + id))); + } + } + + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput); + + List> dropdownOptionList = new ArrayList<>(); + dataList.add(dropdownOptionList); + + ////////////////////////////////////////// + // sort results, dedupe, and add to map // + ////////////////////////////////////////// + Set exists = new HashSet<>(); + output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel())); + for(QPossibleValue possibleValue : output.getResults()) + { + dropdownOptionList.add(MapBuilder.of( + "id", String.valueOf(possibleValue.getId()), + "label", possibleValue.getLabel() + )); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // because we know the dropdowns and what the field names will be when something is selected, we can make // + // sure that something has been selected, and if not, display a message that a selection needs made // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(dropdownData.getIsRequired()) + { + if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName))) + { + missingRequiredSelections.add(pvsLabel); + } + } } } } - widgetData.setDropdownNameList(pvsNames); - widgetData.setDropdownLabelList(pvsLabels); - widgetData.setDropdownDataList(pvsData); + widgetData.setDropdownNameList(nameList); + widgetData.setDropdownLabelList(labelList); + widgetData.setDropdownDataList(dataList); //////////////////////////////////////////////////////////////////////////////// // if there are any missing required dropdowns, build up a message to display // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java new file mode 100644 index 00000000..5a9f67ce --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java @@ -0,0 +1,202 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Generic widget that does an aggregate query, and presents its results + ** as a table, using group-by values as both row & column labels. + *******************************************************************************/ +public class Aggregate2DTableWidgetRenderer extends AbstractWidgetRenderer +{ + private static final QLogger LOG = QLogger.getLogger(Aggregate2DTableWidgetRenderer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + Map values = input.getWidgetMetaData().getDefaultValues(); + + String tableName = ValueUtils.getValueAsString(values.get("tableName")); + String valueField = ValueUtils.getValueAsString(values.get("valueField")); + String rowField = ValueUtils.getValueAsString(values.get("rowField")); + String columnField = ValueUtils.getValueAsString(values.get("columnField")); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + AggregateInput aggregateInput = new AggregateInput(); + aggregateInput.setTableName(tableName); + + // todo - allow input of "list of columns" (e.g., in case some miss sometimes, or as a version of filter) + // todo - max rows, max cols? + + // todo - from input map + QQueryFilter filter = new QQueryFilter(); + aggregateInput.setFilter(filter); + + Aggregate aggregate = new Aggregate(valueField, AggregateOperator.COUNT); + aggregateInput.withAggregate(aggregate); + + GroupBy rowGroupBy = new GroupBy(table.getField(rowField)); + GroupBy columnGroupBy = new GroupBy(table.getField(columnField)); + aggregateInput.withGroupBy(rowGroupBy); + aggregateInput.withGroupBy(columnGroupBy); + + String orderBys = ValueUtils.getValueAsString(values.get("orderBys")); + if(StringUtils.hasContent(orderBys)) + { + for(String orderBy : orderBys.split(",")) + { + switch(orderBy) + { + case "row" -> filter.addOrderBy(new QFilterOrderByGroupBy(rowGroupBy)); + case "column" -> filter.addOrderBy(new QFilterOrderByGroupBy(columnGroupBy)); + case "value" -> filter.addOrderBy(new QFilterOrderByAggregate(aggregate)); + default -> LOG.warn("Unrecognized orderBy: " + orderBy); + } + } + } + + AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); + + Map> data = new LinkedHashMap<>(); + Set columnsSet = new LinkedHashSet<>(); + + for(AggregateResult result : aggregateOutput.getResults()) + { + Serializable column = result.getGroupByValue(columnGroupBy); + Serializable row = result.getGroupByValue(rowGroupBy); + Serializable value = result.getAggregateValue(aggregate); + + Map rowMap = data.computeIfAbsent(row, (k) -> new LinkedHashMap<>()); + rowMap.put(column, value); + columnsSet.add(column); + } + + // todo - possible values from rows, cols + + //////////////////////////////////// + // setup datastructures for table // + //////////////////////////////////// + List> tableRows = new ArrayList<>(); + List tableColumns = new ArrayList<>(); + tableColumns.add(new TableData.Column("default", table.getField(rowField).getLabel(), "_row", "2fr", "left")); + + for(Serializable column : columnsSet) + { + tableColumns.add(new TableData.Column("default", String.valueOf(column) /* todo display value */, String.valueOf(column), "1fr", "right")); + } + + tableColumns.add(new TableData.Column("default", "Total", "_total", "1fr", "right")); + + TableData tableData = new TableData(null, tableColumns, tableRows) + .withRowsPerPage(100) + .withFixedStickyLastRow(false) + .withHidePaginationDropdown(true); + + Map columnSums = new HashMap<>(); + int grandTotal = 0; + for(Map.Entry> rowEntry : data.entrySet()) + { + Map rowMap = new HashMap<>(); + tableRows.add(rowMap); + + rowMap.put("_row", rowEntry.getKey() /* todo display value */); + int rowTotal = 0; + for(Serializable column : columnsSet) + { + Serializable value = rowEntry.getValue().get(column); + if(value == null) + { + value = 0; // todo? + } + + Integer valueAsInteger = Objects.requireNonNullElse(ValueUtils.getValueAsInteger(value), 0); + rowTotal += valueAsInteger; + columnSums.putIfAbsent(column, 0); + columnSums.put(column, columnSums.get(column) + valueAsInteger); + + rowMap.put(String.valueOf(column), value); // todo format commas? + } + + rowMap.put("_total", rowTotal); + grandTotal += rowTotal; + } + + /////////////// + // total row // + /////////////// + Map totalRowMap = new HashMap<>(); + tableRows.add(totalRowMap); + + totalRowMap.put("_row", "Total"); + int rowTotal = 0; + for(Serializable column : columnsSet) + { + Serializable value = columnSums.get(column); + if(value == null) + { + value = 0; // todo? + } + + totalRowMap.put(String.valueOf(column), value); // todo format commas? + } + + totalRowMap.put("_total", grandTotal); + + return (new RenderWidgetOutput(tableData)); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java index b6f69d4c..c940b690 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; @@ -59,6 +60,7 @@ import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang.BooleanUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -66,6 +68,9 @@ import org.apache.commons.lang.BooleanUtils; *******************************************************************************/ public class ChildRecordListRenderer extends AbstractWidgetRenderer { + private static final QLogger LOG = QLogger.getLogger(ChildRecordListRenderer.class); + + /******************************************************************************* ** @@ -137,7 +142,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer *******************************************************************************/ public Builder withCanAddChildRecord(boolean b) { - widgetMetaData.withDefaultValue("canAddChildRecord", true); + widgetMetaData.withDefaultValue("canAddChildRecord", b); return (this); } @@ -151,6 +156,17 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer widgetMetaData.withDefaultValue("disabledFieldsForNewChildRecords", new HashSet<>(disabledFieldsForNewChildRecords)); return (this); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withManageAssociationName(String manageAssociationName) + { + widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName); + return (this); + } } @@ -161,98 +177,134 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer @Override public RenderWidgetOutput render(RenderWidgetInput input) throws QException { - String widgetLabel = input.getQueryParams().get("widgetLabel"); - String joinName = input.getQueryParams().get("joinName"); - QJoinMetaData join = input.getInstance().getJoin(joinName); - String id = input.getQueryParams().get("id"); - QTableMetaData leftTable = input.getInstance().getTable(join.getLeftTable()); - QTableMetaData rightTable = input.getInstance().getTable(join.getRightTable()); - - Integer maxRows = null; - if(StringUtils.hasContent(input.getQueryParams().get("maxRows"))) + try { - maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows")); - } - else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")) - { - maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")); - } + String widgetLabel = input.getQueryParams().get("widgetLabel"); + String joinName = input.getQueryParams().get("joinName"); + QJoinMetaData join = input.getInstance().getJoin(joinName); + String id = input.getQueryParams().get("id"); + QTableMetaData leftTable = input.getInstance().getTable(join.getLeftTable()); + QTableMetaData rightTable = input.getInstance().getTable(join.getRightTable()); - //////////////////////////////////////////////////////// - // fetch the record that we're getting children for. // - // e.g., the left-side of the join, with the input id // - //////////////////////////////////////////////////////// - GetInput getInput = new GetInput(); - getInput.setTableName(join.getLeftTable()); - getInput.setPrimaryKey(id); - GetOutput getOutput = new GetAction().execute(getInput); - QRecord record = getOutput.getRecord(); - - if(record == null) - { - throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id)); - } - - //////////////////////////////////////////////////////////////////// - // set up the query - for the table on the right side of the join // - //////////////////////////////////////////////////////////////////// - QQueryFilter filter = new QQueryFilter(); - for(JoinOn joinOn : join.getJoinOns()) - { - filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField())))); - } - filter.setOrderBys(join.getOrderBys()); - filter.setLimit(maxRows); - - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(join.getRightTable()); - queryInput.setShouldTranslatePossibleValues(true); - queryInput.setShouldGenerateDisplayValues(true); - queryInput.setFilter(filter); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords()); - - int totalRows = queryOutput.getRecords().size(); - if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) - { - ///////////////////////////////////////////////////////////////////////////////////// - // if the input said to only do some max, and the # of results we got is that max, // - // then do a count query, for displaying 1-n of // - ///////////////////////////////////////////////////////////////////////////////////// - CountInput countInput = new CountInput(); - countInput.setTableName(join.getRightTable()); - countInput.setFilter(filter); - totalRows = new CountAction().execute(countInput).getCount(); - } - - String tablePath = input.getInstance().getTablePath(rightTable.getName()); - String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset())); - - ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows); - - if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord")))) - { - widgetData.setCanAddChildRecord(true); - - ////////////////////////////////////////////////////////// - // new child records must have values from the join-ons // - ////////////////////////////////////////////////////////// - Map defaultValuesForNewChildRecords = new HashMap<>(); - for(JoinOn joinOn : join.getJoinOns()) + Integer maxRows = null; + if(StringUtils.hasContent(input.getQueryParams().get("maxRows"))) { - defaultValuesForNewChildRecords.put(joinOn.getRightField(), record.getValue(joinOn.getLeftField())); + maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows")); } - widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords); - - Map widgetValues = input.getWidgetMetaData().getDefaultValues(); - if(widgetValues.containsKey("disabledFieldsForNewChildRecords")) + else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")) { - widgetData.setDisabledFieldsForNewChildRecords((Set) widgetValues.get("disabledFieldsForNewChildRecords")); + maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")); } - } - return (new RenderWidgetOutput(widgetData)); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // fetch the record that we're getting children for. e.g., the left-side of the join, with the input id // + // but - only try this if we were given an id. note, this widget could be called for on an INSERT screen, where we don't have a record yet // + // but we still want to be able to return all the other data in here that otherwise comes from the widget meta data, join, etc. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int totalRows = 0; + QRecord primaryRecord = null; + QQueryFilter filter = null; + QueryOutput queryOutput = new QueryOutput(new QueryInput()); + if(StringUtils.hasContent(id)) + { + GetInput getInput = new GetInput(); + getInput.setTableName(join.getLeftTable()); + getInput.setPrimaryKey(id); + GetOutput getOutput = new GetAction().execute(getInput); + primaryRecord = getOutput.getRecord(); + + if(primaryRecord == null) + { + throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id)); + } + + //////////////////////////////////////////////////////////////////// + // set up the query - for the table on the right side of the join // + //////////////////////////////////////////////////////////////////// + filter = new QQueryFilter(); + for(JoinOn joinOn : join.getJoinOns()) + { + filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(primaryRecord.getValue(joinOn.getLeftField())))); + } + filter.setOrderBys(join.getOrderBys()); + filter.setLimit(maxRows); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(join.getRightTable()); + queryInput.setShouldTranslatePossibleValues(true); + queryInput.setShouldGenerateDisplayValues(true); + queryInput.setFilter(filter); + queryOutput = new QueryAction().execute(queryInput); + + QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords()); + + totalRows = queryOutput.getRecords().size(); + if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) + { + ///////////////////////////////////////////////////////////////////////////////////// + // if the input said to only do some max, and the # of results we got is that max, // + // then do a count query, for displaying 1-n of // + ///////////////////////////////////////////////////////////////////////////////////// + CountInput countInput = new CountInput(); + countInput.setTableName(join.getRightTable()); + countInput.setFilter(filter); + totalRows = new CountAction().execute(countInput).getCount(); + } + } + + String tablePath = input.getInstance().getTablePath(rightTable.getName()); + String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset())); + + ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows); + + if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord")))) + { + widgetData.setCanAddChildRecord(true); + + ////////////////////////////////////////////////////////// + // new child records must have values from the join-ons // + ////////////////////////////////////////////////////////// + Map defaultValuesForNewChildRecords = new HashMap<>(); + if(primaryRecord != null) + { + for(JoinOn joinOn : join.getJoinOns()) + { + defaultValuesForNewChildRecords.put(joinOn.getRightField(), primaryRecord.getValue(joinOn.getLeftField())); + } + } + + widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords); + + Map widgetValues = input.getWidgetMetaData().getDefaultValues(); + if(widgetValues.containsKey("disabledFieldsForNewChildRecords")) + { + widgetData.setDisabledFieldsForNewChildRecords((Set) widgetValues.get("disabledFieldsForNewChildRecords")); + } + else + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are no disabled fields specified - then normally any fields w/ a default value get implicitly disabled // + // but - if we didn't look-up the primary record, then we'll want to explicit disable fields from joins // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(primaryRecord == null) + { + Set implicitlyDisabledFields = new HashSet<>(); + widgetData.setDisabledFieldsForNewChildRecords(implicitlyDisabledFields); + for(JoinOn joinOn : join.getJoinOns()) + { + implicitlyDisabledFields.add(joinOn.getRightField()); + } + } + } + } + + return (new RenderWidgetOutput(widgetData)); + } + catch(Exception e) + { + LOG.warn("Error rendering child record list", e, logPair("widgetName", () -> input.getWidgetMetaData().getName())); + throw (e); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java index 2cc0ba81..47cedbf9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java @@ -256,7 +256,6 @@ public enum DateTimeGroupBy /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") public Instant roundDown(Instant instant, ZoneId zoneId) { ZonedDateTime zoned = instant.atZone(zoneId); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidget.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidget.java new file mode 100644 index 00000000..486f192c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidget.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.AlertData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; + + +/******************************************************************************* + ** Widget that can add an Alert to a process screen. + ** + ** In the process, you'll want values: + ** - alertType - name of entry in AlertType enum (ERROR, WARNING, SUCCESS) + ** - alertHtml - html to display inside the alert (other than its icon) + *******************************************************************************/ +public class ProcessAlertWidget extends AbstractWidgetRenderer implements MetaDataProducerInterface +{ + public static final String NAME = "ProcessAlertWidget"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + AlertData.AlertType alertType = AlertData.AlertType.WARNING; + if(input.getQueryParams().containsKey("alertType")) + { + alertType = AlertData.AlertType.valueOf(input.getQueryParams().get("alertType")); + } + + String html = "Warning"; + if(input.getQueryParams().containsKey("alertHtml")) + { + html = input.getQueryParams().get("alertHtml"); + } + + return (new RenderWidgetOutput(new AlertData(alertType, html))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QWidgetMetaData produce(QInstance qInstance) throws QException + { + return new QWidgetMetaData() + .withType(WidgetType.ALERT.getType()) + .withGridColumns(12) + .withName(NAME) + .withIsCard(false) + .withShowReloadButton(false) + .withCodeReference(new QCodeReference(getClass())); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java new file mode 100644 index 00000000..99b7462d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.interfaces; + + +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; + + +/******************************************************************************* + ** Interface for actions that a backend can perform, based on streaming data + ** into the backend's storage. + *******************************************************************************/ +public interface QStorageInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + OutputStream createOutputStream(StorageInput storageInput) throws QException; + + + /******************************************************************************* + ** + *******************************************************************************/ + InputStream getInputStream(StorageInput storageInput) throws QException; + + + /******************************************************************************* + ** + *******************************************************************************/ + default void makePublic(StorageInput storageInput) throws QException + { + ////////// + // noop // + ////////// + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default String getDownloadURL(StorageInput storageInput) throws QException + { + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/messaging/SendMessageAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/messaging/SendMessageAction.java new file mode 100644 index 00000000..f0cccf1e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/messaging/SendMessageAction.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.messaging; + + +import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput; +import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData; +import com.kingsrook.qqq.backend.core.modules.messaging.MessagingProviderInterface; +import com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SendMessageAction extends AbstractQActionFunction +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SendMessageOutput execute(SendMessageInput input) throws QException + { + if(!StringUtils.hasContent(input.getMessagingProviderName())) + { + throw (new QException("Messaging provider name was not given in SendMessageInput.")); + } + + QMessagingProviderMetaData messagingProvider = QContext.getQInstance().getMessagingProvider(input.getMessagingProviderName()); + if(messagingProvider == null) + { + throw (new QException("Messaging provider named [" + input.getMessagingProviderName() + "] was not found in this QInstance.")); + } + + MessagingProviderInterface messagingProviderInterface = new QMessagingProviderDispatcher().getMessagingProviderInterface(messagingProvider.getType()); + + return (messagingProviderInterface.sendMessage(input)); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java index fc597ed7..52619b36 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -41,9 +42,19 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class JoinGraph { - private Set edges = new HashSet<>(); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // as an instance grows, with the number of joins (say, more than 50?), especially as they may have a lot of connections, // + // it can become very very slow to process a full join graph (e.g., 10 seconds, maybe much worse, per Big-O...) // + // also, it's not frequently useful to look at a join path that's more than a handful of tables long. // + // thus - this property exists - to limit the max length of a join path. Keeping it small keeps instance enrichment // + // and validation reasonably performant, at the possible cost of, some join-path that's longer than this limit may not // + // be found - but - chances are, you don't want some 12-element join path to be used anyway, thus, this makes sense. // + // but - it can be adjusted, per system property or ENV var. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + private int maxPathLength = new QMetaDataVariableInterpreter().getIntegerFromPropertyOrEnvironment("qqq.instance.joinGraph.maxPathLength", "QQQ_INSTANCE_JOIN_GRAPH_MAX_PATH_LENGTH", 3); + /******************************************************************************* @@ -303,6 +314,13 @@ public class JoinGraph if(otherTableName != null) { + if(newPath.size() > maxPathLength) + { + //////////////////////////////////////////////////////////////// + // performance hack. see comment at maxPathLength definition // + //////////////////////////////////////////////////////////////// + continue; + } JoinConnectionList newConnectionList = connectionList.copy(); JoinConnection joinConnection = new JoinConnection(otherTableName, edge.joinName); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java index 39090e2f..ab1b56ca 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java @@ -34,15 +34,18 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -333,9 +336,25 @@ public class PermissionsHelper QPermissionRules rules = getEffectivePermissionRules(tableMetaData, instance); String baseName = getEffectivePermissionBaseName(rules, tableMetaData.getName()); - for(TablePermissionSubType permissionSubType : TablePermissionSubType.values()) + QBackendMetaData backend = instance.getBackend(tableMetaData.getBackendName()); + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_INSERT)) { - addEffectiveAvailablePermission(rules, permissionSubType, rs, baseName, tableMetaData, "Table"); + addEffectiveAvailablePermission(rules, TablePermissionSubType.INSERT, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_UPDATE)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.EDIT, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_DELETE)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.DELETE, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_QUERY) || tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_GET)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.READ, rs, baseName, tableMetaData, "Table"); } } @@ -369,7 +388,10 @@ public class PermissionsHelper { QPermissionRules rules = getEffectivePermissionRules(widgetMetaData, instance); String baseName = getEffectivePermissionBaseName(rules, widgetMetaData.getName()); - addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget"); + if(!rules.getLevel().equals(PermissionLevel.NOT_PROTECTED)) + { + addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget"); + } } return (rs); @@ -478,7 +500,6 @@ public class PermissionsHelper /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") static PermissionSubType getEffectivePermissionSubType(QPermissionRules rules, PermissionSubType originalPermissionSubType) { if(rules == null || rules.getLevel() == null) @@ -493,10 +514,10 @@ public class PermissionsHelper if(PrivatePermissionSubType.HAS_ACCESS.equals(originalPermissionSubType)) { return switch(rules.getLevel()) - { - case NOT_PROTECTED -> null; - default -> PrivatePermissionSubType.HAS_ACCESS; - }; + { + case NOT_PROTECTED -> null; + default -> PrivatePermissionSubType.HAS_ACCESS; + }; } else { @@ -505,30 +526,30 @@ public class PermissionsHelper // permission sub-type to what we expect to be set for the table // //////////////////////////////////////////////////////////////////////////////////////////////////////// return switch(rules.getLevel()) + { + case NOT_PROTECTED -> null; + case HAS_ACCESS_PERMISSION -> PrivatePermissionSubType.HAS_ACCESS; + case READ_WRITE_PERMISSIONS -> { - case NOT_PROTECTED -> null; - case HAS_ACCESS_PERMISSION -> PrivatePermissionSubType.HAS_ACCESS; - case READ_WRITE_PERMISSIONS -> + if(PrivatePermissionSubType.READ.equals(originalPermissionSubType) || PrivatePermissionSubType.WRITE.equals(originalPermissionSubType)) { - if(PrivatePermissionSubType.READ.equals(originalPermissionSubType) || PrivatePermissionSubType.WRITE.equals(originalPermissionSubType)) - { - yield (originalPermissionSubType); - } - else if(TablePermissionSubType.INSERT.equals(originalPermissionSubType) || TablePermissionSubType.EDIT.equals(originalPermissionSubType) || TablePermissionSubType.DELETE.equals(originalPermissionSubType)) - { - yield (PrivatePermissionSubType.WRITE); - } - else if(TablePermissionSubType.READ.equals(originalPermissionSubType)) - { - yield (PrivatePermissionSubType.READ); - } - else - { - throw new IllegalStateException("Unexpected permissionSubType: " + originalPermissionSubType); - } + yield (originalPermissionSubType); } - case READ_INSERT_EDIT_DELETE_PERMISSIONS -> originalPermissionSubType; - }; + else if(TablePermissionSubType.INSERT.equals(originalPermissionSubType) || TablePermissionSubType.EDIT.equals(originalPermissionSubType) || TablePermissionSubType.DELETE.equals(originalPermissionSubType)) + { + yield (PrivatePermissionSubType.WRITE); + } + else if(TablePermissionSubType.READ.equals(originalPermissionSubType)) + { + yield (PrivatePermissionSubType.READ); + } + else + { + throw new IllegalStateException("Unexpected permissionSubType: " + originalPermissionSubType); + } + } + case READ_INSERT_EDIT_DELETE_PERMISSIONS -> originalPermissionSubType; + }; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessAction.java new file mode 100644 index 00000000..1e70f88c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessAction.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.processes; + + +import java.util.Optional; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.state.StateType; +import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Action handler for running the cancel step of a qqq process + * + *******************************************************************************/ +public class CancelProcessAction extends RunProcessAction +{ + private static final QLogger LOG = QLogger.getLogger(CancelProcessAction.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public RunProcessOutput execute(RunProcessInput runProcessInput) throws QException + { + ActionHelper.validateSession(runProcessInput); + + QProcessMetaData process = runProcessInput.getInstance().getProcess(runProcessInput.getProcessName()); + if(process == null) + { + throw new QBadRequestException("Process [" + runProcessInput.getProcessName() + "] is not defined in this instance."); + } + + if(runProcessInput.getProcessUUID() == null) + { + throw (new QBadRequestException("Cannot cancel process - processUUID was not given.")); + } + + UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS); + Optional processState = getState(runProcessInput.getProcessUUID()); + if(processState.isEmpty()) + { + throw (new QBadRequestException("Cannot cancel process - State for process UUID [" + runProcessInput.getProcessUUID() + "] was not found.")); + } + + RunProcessOutput runProcessOutput = new RunProcessOutput(); + try + { + if(process.getCancelStep() != null) + { + LOG.info("Running cancel step for process", logPair("processName", process.getName())); + runBackendStep(runProcessInput, process, runProcessOutput, stateKey, process.getCancelStep(), process, processState.get()); + } + else + { + LOG.debug("Process does not have a custom cancel step to run.", logPair("processName", process.getName())); + } + } + catch(QException qe) + { + //////////////////////////////////////////////////////////// + // upon exception (e.g., one thrown by a step), throw it. // + //////////////////////////////////////////////////////////// + throw (qe); + } + catch(Exception e) + { + throw (new QException("Error cancelling process", e)); + } + finally + { + ////////////////////////////////////////////////////// + // always put the final state in the process result // + ////////////////////////////////////////////////////// + runProcessOutput.setProcessState(processState.get()); + } + + return (runProcessOutput); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java index 977104c0..a61fb031 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java @@ -23,11 +23,19 @@ package com.kingsrook.qqq.backend.core.actions.processes; import java.io.Serializable; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -65,4 +73,56 @@ public class QProcessCallbackFactory }; } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessCallback forRecordEntity(QRecordEntity entity) + { + return forRecord(entity.toQRecord()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessCallback forRecord(QRecord record) + { + String primaryKeyField = "id"; + if(StringUtils.hasContent(record.getTableName())) + { + primaryKeyField = QContext.getQInstance().getTable(record.getTableName()).getPrimaryKeyField(); + } + + Serializable primaryKeyValue = record.getValue(primaryKeyField); + if(primaryKeyValue == null) + { + throw (new QRuntimeException("Record did not have value in its primary key field [" + primaryKeyField + "]")); + } + + return (forPrimaryKey(primaryKeyField, primaryKeyValue)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessCallback forPrimaryKey(String fieldName, Serializable value) + { + return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value)))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessCallback forPrimaryKeys(String fieldName, Collection values) + { + return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IN, values)))); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java index fc23904d..8b0d7dd3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; @@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -71,7 +73,17 @@ public class RunBackendStepAction QStepMetaData stepMetaData = process.getStep(runBackendStepInput.getStepName()); if(stepMetaData == null) { - throw new QException("Step [" + runBackendStepInput.getStepName() + "] is not defined in the process [" + process.getName() + "]"); + if(process.getCancelStep() != null && Objects.equals(process.getCancelStep().getName(), runBackendStepInput.getStepName())) + { + ///////////////////////////////////// + // special case for cancel step... // + ///////////////////////////////////// + stepMetaData = process.getCancelStep(); + } + else + { + throw new QException("Step [" + runBackendStepInput.getStepName() + "] is not defined in the process [" + process.getName() + "]"); + } } if(!(stepMetaData instanceof QBackendStepMetaData backendStepMetaData)) @@ -82,7 +94,7 @@ public class RunBackendStepAction ////////////////////////////////////////////////////////////////////////////////////// // ensure input data is set as needed - use callback object to get anything missing // ////////////////////////////////////////////////////////////////////////////////////// - ensureRecordsAreInRequest(runBackendStepInput, backendStepMetaData); + ensureRecordsAreInRequest(runBackendStepInput, backendStepMetaData, process); ensureInputFieldsAreInRequest(runBackendStepInput, backendStepMetaData); //////////////////////////////////////////////////////////////////// @@ -167,7 +179,7 @@ public class RunBackendStepAction ** check if this step uses a record list - and if so, if we need to get one ** via the callback *******************************************************************************/ - private void ensureRecordsAreInRequest(RunBackendStepInput runBackendStepInput, QBackendStepMetaData step) throws QException + private void ensureRecordsAreInRequest(RunBackendStepInput runBackendStepInput, QBackendStepMetaData step, QProcessMetaData process) throws QException { QFunctionInputMetaData inputMetaData = step.getInputMetaData(); if(inputMetaData != null && inputMetaData.getRecordListMetaData() != null) @@ -190,9 +202,44 @@ public class RunBackendStepAction queryInput.setFilter(callback.getQueryFilter()); + ////////////////////////////////////////////////////////////////////////////////////////// + // if process has a max-no of records, set a limit on the process of that number plus 1 // + // (the plus 1 being so we can see "oh, you selected more than that many; error!" // + ////////////////////////////////////////////////////////////////////////////////////////// + if(process.getMaxInputRecords() != null) + { + if(callback.getQueryFilter() == null) + { + queryInput.setFilter(new QQueryFilter()); + } + + queryInput.getFilter().setLimit(process.getMaxInputRecords() + 1); + } + QueryOutput queryOutput = new QueryAction().execute(queryInput); runBackendStepInput.setRecords(queryOutput.getRecords()); - // todo - handle 0 results found? + + //////////////////////////////////////////////////////////////////////////////// + // if process defines a max, and more than the max were found, throw an error // + //////////////////////////////////////////////////////////////////////////////// + if(process.getMaxInputRecords() != null) + { + if(queryOutput.getRecords().size() > process.getMaxInputRecords()) + { + throw (new QUserFacingException("Too many records were selected for this process. At most, only " + process.getMaxInputRecords() + " can be selected.")); + } + } + + ///////////////////////////////////////////////////////////////////////////////// + // if process defines a min, and fewer than the min were found, throw an error // + ///////////////////////////////////////////////////////////////////////////////// + if(process.getMinInputRecords() != null) + { + if(queryOutput.getRecords().size() < process.getMinInputRecords()) + { + throw (new QUserFacingException("Too few records were selected for this process. At least " + process.getMinInputRecords() + " must be selected.")); + } + } } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index c9350e68..8624b926 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -82,7 +82,7 @@ public class RunProcessAction public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey"; public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey"; public static final String BASEPULL_TIMESTAMP_FIELD = "basepullTimestampField"; - public static final String BASEPULL_CONFIGURATION = "basepullConfiguration"; + public static final String BASEPULL_CONFIGURATION = "basepullConfiguration"; //////////////////////////////////////////////////////////////////////////////////////////////// // indicator that the timestamp field should be updated - e.g., the execute step is finished. // @@ -190,7 +190,25 @@ public class RunProcessAction // Run backend steps // /////////////////////// LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]"); - runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); + RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); + + ///////////////////////////////////////////////////////////////////////////////////////// + // if the step returned an override lastStepName, use that to determine how we proceed // + ///////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getOverrideLastStepName() != null) + { + LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!"); + lastStepName = runBackendStepOutput.getOverrideLastStepName(); + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // similarly, if the step produced an updatedFrontendStepList, propagate that data outward // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getUpdatedFrontendStepList() != null) + { + LOG.debug("Process step [" + lastStepName + "] generated an updatedFrontendStepList [" + runBackendStepOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList() + "]!"); + runProcessOutput.setUpdatedFrontendStepList(runBackendStepOutput.getUpdatedFrontendStepList()); + } } else { @@ -317,6 +335,13 @@ public class RunProcessAction /////////////////////////////////////////////////// runProcessInput.seedFromProcessState(optionalProcessState.get()); + /////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're restoring an old state, we can discard a previously stored updatedFrontendStepList - // + // it is only needed on the transitional edge from a backend-step to a frontend step, but not // + // in the other directly // + /////////////////////////////////////////////////////////////////////////////////////////////////// + optionalProcessState.get().setUpdatedFrontendStepList(null); + /////////////////////////////////////////////////////////////////////////// // if there were values from the caller, put those (back) in the request // /////////////////////////////////////////////////////////////////////////// @@ -339,7 +364,7 @@ public class RunProcessAction /******************************************************************************* ** Run a single backend step. *******************************************************************************/ - private void runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception + protected RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception { RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState); runBackendStepInput.setProcessName(process.getName()); @@ -368,14 +393,16 @@ public class RunProcessAction runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY)); } - RunBackendStepOutput lastFunctionResult = new RunBackendStepAction().execute(runBackendStepInput); - storeState(stateKey, lastFunctionResult.getProcessState()); + RunBackendStepOutput runBackendStepOutput = new RunBackendStepAction().execute(runBackendStepInput); + storeState(stateKey, runBackendStepOutput.getProcessState()); - if(lastFunctionResult.getException() != null) + if(runBackendStepOutput.getException() != null) { - runProcessOutput.setException(lastFunctionResult.getException()); - throw (lastFunctionResult.getException()); + runProcessOutput.setException(runBackendStepOutput.getException()); + throw (runBackendStepOutput.getException()); } + + return (runBackendStepOutput); } @@ -495,15 +522,15 @@ public class RunProcessAction String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName(); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if backend specifies that it uses variants, look for that data in the session and append to our basepull key // + // if process specifies that it uses variants, look for that data in the session and append to our basepull key // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(process.getSchedule() != null && process.getSchedule().getVariantBackend() != null) + if(process.getVariantBackend() != null) { QSession session = QContext.getQSession(); - QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getSchedule().getVariantBackend()); + QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend()); if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) { - LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"); + LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"); } else { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java index 49ecd772..fac33e13 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -65,12 +66,12 @@ public class CsvExportStreamer implements ExportStreamerInterface ** *******************************************************************************/ @Override - public void start(ExportInput exportInput, List fields, String label) throws QReportingException + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException { this.exportInput = exportInput; this.fields = fields; table = exportInput.getTable(); - outputStream = this.exportInput.getReportOutputStream(); + outputStream = this.exportInput.getReportDestination().getReportOutputStream(); writeTitleAndHeader(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/DistinctFilteringRecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/DistinctFilteringRecordPipe.java new file mode 100644 index 00000000..ad2aa3fd --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/DistinctFilteringRecordPipe.java @@ -0,0 +1,146 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; + + +/******************************************************************************* + ** Subclass of record pipe that ony allows through distinct records, based on + ** the set of fields specified in the constructor as a uniqueKey. + *******************************************************************************/ +public class DistinctFilteringRecordPipe extends RecordPipe +{ + private UniqueKey uniqueKey; + private Set seenValues = new HashSet<>(); + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public DistinctFilteringRecordPipe(UniqueKey uniqueKey) + { + this.uniqueKey = uniqueKey; + } + + + + /******************************************************************************* + ** Constructor that accepts pipe's overrideCapacity (allowed to be null) + ** + *******************************************************************************/ + public DistinctFilteringRecordPipe(UniqueKey uniqueKey, Integer overrideCapacity) + { + super(overrideCapacity); + this.uniqueKey = uniqueKey; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addRecords(List records) throws QException + { + List recordsToAdd = new ArrayList<>(); + for(QRecord record : records) + { + if(!seenBefore(record)) + { + recordsToAdd.add(record); + } + } + + if(recordsToAdd.isEmpty()) + { + return; + } + + super.addRecords(recordsToAdd); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addRecord(QRecord record) throws QException + { + if(seenBefore(record)) + { + return; + } + + super.addRecord(record); + } + + + + /******************************************************************************* + ** return true if we've seen this record before (based on the unique key) - + ** also - update the set of seen values! + *******************************************************************************/ + private boolean seenBefore(QRecord record) + { + Serializable ukValues = extractUKValues(record); + if(seenValues.contains(ukValues)) + { + return true; + } + seenValues.add(ukValues); + return false; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Serializable extractUKValues(QRecord record) + { + if(uniqueKey.getFieldNames().size() == 1) + { + return (record.getValue(uniqueKey.getFieldNames().get(0))); + } + else + { + ArrayList rs = new ArrayList<>(); + for(String fieldName : uniqueKey.getFieldNames()) + { + rs.add(record.getValue(fieldName)); + } + return (rs); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java index 6bd4b83d..e14de682 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java @@ -52,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; @@ -138,7 +139,7 @@ public class ExportAction /////////////////////////////////////////////////////////////////////////////////////////////////////////// // check if this report format has a max-rows limit -- if so, do a count to verify we're under the limit // /////////////////////////////////////////////////////////////////////////////////////////////////////////// - ReportFormat reportFormat = exportInput.getReportFormat(); + ReportFormat reportFormat = exportInput.getReportDestination().getReportFormat(); verifyCountUnderMax(exportInput, backendModule, reportFormat); preExecuteRan = true; @@ -189,6 +190,9 @@ public class ExportAction Set addedJoinNames = new HashSet<>(); if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames())) { + ///////////////////////////////////////////////////////////////////////////////////////////// + // make sure that any tables being selected from are included as (LEFT) joins in the query // + ///////////////////////////////////////////////////////////////////////////////////////////// for(String fieldName : exportInput.getFieldNames()) { if(fieldName.contains(".")) @@ -197,27 +201,7 @@ public class ExportAction String joinTableName = parts[0]; if(!addedJoinNames.contains(joinTableName)) { - QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true); - queryJoins.add(queryJoin); - - ///////////////////////////////////////////////////////////////////////////////////////////// - // in at least some cases, we need to let the queryJoin know what join-meta-data to use... // - // This code basically mirrors what QFMD is doing right now, so it's better - // - // but shouldn't all of this just be in JoinsContext? it does some of this... // - ///////////////////////////////////////////////////////////////////////////////////////////// - QTableMetaData table = exportInput.getTable(); - Optional exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst(); - if(exposedJoinOptional.isEmpty()) - { - throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName)); - } - ExposedJoin exposedJoin = exposedJoinOptional.get(); - - if(exposedJoin.getJoinPath().size() == 1) - { - queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1))); - } - + queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true)); addedJoinNames.add(joinTableName); } } @@ -232,6 +216,7 @@ public class ExportAction } queryInput.getFilter().setLimit(exportInput.getLimit()); queryInput.setShouldTranslatePossibleValues(true); + queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS); ///////////////////////////////////////////////////////////////// // tell this query that it needs to put its output into a pipe // @@ -242,10 +227,19 @@ public class ExportAction //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up a report streamer, which will read rows from the pipe, and write formatted report rows to the output stream // //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ReportFormat reportFormat = exportInput.getReportFormat(); + ReportFormat reportFormat = exportInput.getReportDestination().getReportFormat(); ExportStreamerInterface reportStreamer = reportFormat.newReportStreamer(); List fields = getFields(exportInput); - reportStreamer.start(exportInput, fields, "Sheet 1"); + + ////////////////////////////////////////////////////////// + // it seems we can pass a view with just a name in here // + ////////////////////////////////////////////////////////// + List views = new ArrayList<>(); + views.add(new QReportView() + .withName("export")); + + reportStreamer.preRun(exportInput.getReportDestination(), views); + reportStreamer.start(exportInput, fields, "Sheet 1", views.get(0)); ////////////////////////////////////////// // run the query action as an async job // @@ -334,7 +328,7 @@ public class ExportAction try { - exportInput.getReportOutputStream().close(); + exportInput.getReportDestination().getReportOutputStream().close(); } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java index 473b3b34..4e2f7e02 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java @@ -26,8 +26,10 @@ import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; /******************************************************************************* @@ -35,20 +37,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; *******************************************************************************/ public interface ExportStreamerInterface { - /******************************************************************************* - ** Called once, before any rows are available. Meant to write a header, for example. - *******************************************************************************/ - void start(ExportInput exportInput, List fields, String label) throws QReportingException; /******************************************************************************* - ** Called as records flow into the pipe. - ******************************************************************************/ - void addRecords(List recordList) throws QReportingException; - - /******************************************************************************* - ** Called once, after all rows are available. Meant to write a footer, or close resources, for example. + ** Called once, before any sheets are actually being produced. *******************************************************************************/ - void finish() throws QReportingException; + default void preRun(ReportDestination reportDestination, List views) throws QReportingException + { + // noop in base class + } /******************************************************************************* ** @@ -58,6 +54,20 @@ public interface ExportStreamerInterface // noop in base class } + /******************************************************************************* + ** Called once per sheet, before any rows are available. Meant to write a + ** header, for example. + ** + ** If multiple sheets are being created, there is no separate end-sheet call. + ** Rather, a new one will just get started... + *******************************************************************************/ + void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException; + + /******************************************************************************* + ** Called as records flow into the pipe. + ******************************************************************************/ + void addRecords(List recordList) throws QReportingException; + /******************************************************************************* ** *******************************************************************************/ @@ -65,4 +75,11 @@ public interface ExportStreamerInterface { addRecords(List.of(record)); } + + /******************************************************************************* + ** Called after all sheets are complete. Meant to do a final write, or close + ** resources, for example. + *******************************************************************************/ + void finish() throws QReportingException; + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index 93b9d183..3c2ace5c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.io.Serializable; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -32,18 +34,23 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction; import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer; import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QFormulaException; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -51,6 +58,9 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -68,11 +78,17 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface; import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.StringAggregates; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -87,7 +103,7 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; ** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables), ** - native pivot tables (not initially supported, due to lack of support in fastexcel...). *******************************************************************************/ -public class GenerateReportAction +public class GenerateReportAction extends AbstractQActionFunction { private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class); @@ -101,50 +117,67 @@ public class GenerateReportAction // Aggregates: (count:47;sum:10,000;max:2,000;min:15) // // salesSummaryReport > [(state:MO),(city:St.Louis)] > salePrice > (count:47;sum:10,000;max:2,000;min:15) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////// - Map>>> summaryAggregates = new HashMap<>(); - Map>>> varianceAggregates = new HashMap<>(); + Map>>> summaryAggregates = new HashMap<>(); + Map>>> varianceAggregates = new HashMap<>(); - Map> totalAggregates = new HashMap<>(); - Map> varianceTotalAggregates = new HashMap<>(); + Map> totalAggregates = new HashMap<>(); + Map> varianceTotalAggregates = new HashMap<>(); - private QReportMetaData report; - private ReportFormat reportFormat; private ExportStreamerInterface reportStreamer; + private List dataSources; + private List views; + + private Map countByDataSource = new HashMap<>(); /******************************************************************************* ** *******************************************************************************/ - public void execute(ReportInput reportInput) throws QException + public ReportOutput execute(ReportInput reportInput) throws QException { - report = reportInput.getInstance().getReport(reportInput.getReportName()); - reportFormat = reportInput.getReportFormat(); + ReportOutput reportOutput = new ReportOutput(); + QReportMetaData report = getReportMetaData(reportInput); + + this.views = report.getViews(); + this.dataSources = report.getDataSources(); + + ReportFormat reportFormat = reportInput.getReportDestination().getReportFormat(); if(reportFormat == null) { throw new QException("Report format was not specified."); } - reportStreamer = reportFormat.newReportStreamer(); + + if(reportInput.getOverrideExportStreamerSupplier() != null) + { + reportStreamer = reportInput.getOverrideExportStreamerSupplier().get(); + } + else + { + reportStreamer = reportFormat.newReportStreamer(); + } + + reportStreamer.preRun(reportInput.getReportDestination(), views); //////////////////////////////////////////////////////////////////////////////////////////////// // foreach data source, do a query (possibly more than 1, if it goes to multiple table views) // //////////////////////////////////////////////////////////////////////////////////////////////// - for(QReportDataSource dataSource : report.getDataSources()) + for(QReportDataSource dataSource : dataSources) { ////////////////////////////////////////////////////////////////////////////// // make a list of the views that use this data source for various purposes. // ////////////////////////////////////////////////////////////////////////////// - List dataSourceTableViews = report.getViews().stream() + List dataSourceTableViews = views.stream() .filter(v -> v.getType().equals(ReportType.TABLE)) .filter(v -> v.getDataSourceName().equals(dataSource.getName())) .toList(); - List dataSourceSummaryViews = report.getViews().stream() + List dataSourceSummaryViews = views.stream() .filter(v -> v.getType().equals(ReportType.SUMMARY)) .filter(v -> v.getDataSourceName().equals(dataSource.getName())) .toList(); - List dataSourceVariantViews = report.getViews().stream() + List dataSourceVariantViews = views.stream() .filter(v -> v.getType().equals(ReportType.SUMMARY)) .filter(v -> v.getVarianceDataSourceName() != null && v.getVarianceDataSourceName().equals(dataSource.getName())) .toList(); @@ -183,24 +216,50 @@ public class GenerateReportAction //////////////////////////////////////////////////////////////////////////////////// // start the table-view (e.g., open this tab in xlsx) and then run the query-loop // //////////////////////////////////////////////////////////////////////////////////// - startTableView(reportInput, dataSource, dataSourceTableView); + startTableView(reportInput, dataSource, dataSourceTableView, reportFormat); gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews); } } } + ////////////////////// + // add pivot sheets // + ////////////////////// + for(QReportView view : views) + { + if(view.getType().equals(ReportType.PIVOT)) + { + if(reportFormat.getSupportsNativePivotTables()) + { + startTableView(reportInput, null, view, reportFormat); + } + else + { + LOG.warn("Request to render a report with a PIVOT type view, for a format that does not support native pivot tables", logPair("reportFormat", reportFormat)); + } + + ////////////////////////////////////////////////////////////////////////// + // there's no data to add to a pivot table, so nothing else to do here. // + ////////////////////////////////////////////////////////////////////////// + } + } + outputSummaries(reportInput); + reportOutput.setTotalRecordCount(countByDataSource.values().stream().mapToInt(Integer::intValue).sum()); + reportStreamer.finish(); try { - reportInput.getReportOutputStream().close(); + reportInput.getReportDestination().getReportOutputStream().close(); } catch(Exception e) { throw (new QReportingException("Error completing report", e)); } + + return (reportOutput); } @@ -208,28 +267,48 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException + private QReportMetaData getReportMetaData(ReportInput reportInput) throws QException { - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); + if(reportInput.getReportMetaData() != null) + { + return reportInput.getReportMetaData(); + } + if(StringUtils.hasContent(reportInput.getReportName())) + { + return QContext.getQInstance().getReport(reportInput.getReportName()); + } + + throw (new QReportingException("ReportInput did not contain required parameters to identify the report being generated")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView, ReportFormat reportFormat) throws QException + { QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter(); variableInterpreter.addValueMap("input", reportInput.getInputValues()); ExportInput exportInput = new ExportInput(); - exportInput.setReportFormat(reportFormat); - exportInput.setFilename(reportInput.getFilename()); + exportInput.setReportDestination(reportInput.getReportDestination()); exportInput.setTitleRow(getTitle(reportView, variableInterpreter)); exportInput.setIncludeHeaderRow(reportView.getIncludeHeaderRow()); - exportInput.setReportOutputStream(reportInput.getReportOutputStream()); JoinsContext joinsContext = null; - if(StringUtils.hasContent(dataSource.getSourceTable())) + if(dataSource != null) { - joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter()); + if(StringUtils.hasContent(dataSource.getSourceTable())) + { + joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter()); + countDataSourceRecords(reportInput, dataSource, reportFormat); + } } List fields = new ArrayList<>(); - for(QReportField column : reportView.getColumns()) + for(QReportField column : CollectionUtils.nonNullList(reportView.getColumns())) { if(column.getIsVirtual()) { @@ -241,7 +320,7 @@ public class GenerateReportAction JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext == null ? null : joinsContext.getFieldAndTableNameOrAlias(effectiveFieldName); if(fieldAndTableNameOrAlias == null || fieldAndTableNameOrAlias.field() == null) { - throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + dataSource.getName() + "]"); + throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + (dataSource == null ? null : dataSource.getName()) + "]"); } QFieldMetaData field = fieldAndTableNameOrAlias.field().clone(); @@ -255,7 +334,7 @@ public class GenerateReportAction } reportStreamer.setDisplayFormats(getDisplayFormatMap(fields)); - reportStreamer.start(exportInput, fields, reportView.getLabel()); + reportStreamer.start(exportInput, fields, reportView.getLabel(), reportView); } @@ -263,7 +342,36 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List summaryViews, List variantViews) throws QException + private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException + { + QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone(); + setInputValuesInQueryFilter(reportInput, queryFilter); + + CountInput countInput = new CountInput(); + countInput.setTableName(dataSource.getSourceTable()); + countInput.setFilter(queryFilter); + countInput.setQueryJoins(dataSource.getQueryJoins()); + CountOutput countOutput = new CountAction().execute(countInput); + + if(countOutput.getCount() != null) + { + countByDataSource.put(dataSource.getName(), countOutput.getCount()); + + if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows()) + { + throw (new QUserFacingException("The requested report would include more rows (" + + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed (" + + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ").")); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Integer gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List summaryViews, List variantViews) throws QException { //////////////////////////////////////////////////////////////////////////////////////// // check if this view has a transform step - if so, set it up now and run its pre-run // @@ -290,6 +398,9 @@ public class GenerateReportAction RunBackendStepInput finalTransformStepInput = transformStepInput; RunBackendStepOutput finalTransformStepOutput = transformStepOutput; + String tableLabel = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), "")); + AtomicInteger consumedCount = new AtomicInteger(0); + ///////////////////////////////////////////////////////////////// // run a record pipe loop, over the query for this data source // ///////////////////////////////////////////////////////////////// @@ -306,6 +417,7 @@ public class GenerateReportAction queryInput.setTableName(dataSource.getSourceTable()); queryInput.setFilter(queryFilter); queryInput.setQueryJoins(dataSource.getQueryJoins()); + queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS); queryInput.setShouldTranslatePossibleValues(true); queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource, new JoinsContext(reportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryInput.getFilter()))); @@ -345,10 +457,21 @@ public class GenerateReportAction if(finalTransformStep != null) { finalTransformStepInput.setRecords(records); - finalTransformStep.run(finalTransformStepInput, finalTransformStepOutput); + finalTransformStep.runOnePage(finalTransformStepInput, finalTransformStepOutput); records = finalTransformStepOutput.getRecords(); } + Integer total = countByDataSource.get(dataSource.getName()); + if(total != null) + { + reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records", consumedCount.get() + 1, total); + } + else + { + reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records (" + String.format("%,d", consumedCount.get() + 1) + ")"); + } + consumedCount.getAndAdd(records.size()); + return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews)); }); @@ -359,6 +482,8 @@ public class GenerateReportAction { transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput)); } + + return consumedCount.get(); } @@ -366,11 +491,11 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private Set setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) + private Set setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException { Set fieldsToTranslatePossibleValues = new HashSet<>(); - for(QReportView view : report.getViews()) + for(QReportView view : views) { for(QReportField column : CollectionUtils.nonNullList(view.getColumns())) { @@ -384,15 +509,16 @@ public class GenerateReportAction } } - for(String summaryField : CollectionUtils.nonNullList(view.getPivotFields())) + for(String summaryFieldName : CollectionUtils.nonNullList(view.getSummaryFields())) { /////////////////////////////////////////////////////////////////////////////// // all pivotFields that are possible value sources are implicitly translated // /////////////////////////////////////////////////////////////////////////////// - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); - if(table.getField(summaryField).getPossibleValueSourceName() != null) + QTableMetaData mainTable = QContext.getQInstance().getTable(dataSource.getSourceTable()); + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName); + if(fieldAndJoinTable.field().getPossibleValueSourceName() != null) { - fieldsToTranslatePossibleValues.add(summaryField); + fieldsToTranslatePossibleValues.add(summaryFieldName); } } } @@ -405,7 +531,33 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter) + public static FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException + { + if(fieldName.indexOf('.') > -1) + { + String joinTableName = fieldName.replaceAll("\\..*", ""); + String joinFieldName = fieldName.replaceAll(".*\\.", ""); + + QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName); + if(joinTable == null) + { + throw (new QException("Unrecognized join table name: " + joinTableName)); + } + + return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable); + } + else + { + return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter) throws QException { if(queryFilter == null || queryFilter.getCriteria() == null) { @@ -494,26 +646,27 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List records, Map>>> aggregatesMap) + private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List records, Map>>> aggregatesMap) throws QException { - Map>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>()); + Map>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>()); for(QRecord record : records) { SummaryKey key = new SummaryKey(); - for(String summaryField : view.getPivotFields()) + for(String summaryFieldName : view.getSummaryFields()) { - Serializable summaryValue = record.getValue(summaryField); - if(table.getField(summaryField).getPossibleValueSourceName() != null) + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName); + Serializable summaryValue = record.getValue(summaryFieldName); + if(fieldAndJoinTable.field().getPossibleValueSourceName() != null) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - summaryValue = record.getDisplayValue(summaryField); + summaryValue = record.getDisplayValue(summaryFieldName); } - key.add(summaryField, summaryValue); + key.add(summaryFieldName, summaryValue); - if(view.getIncludePivotSubTotals() && key.getKeys().size() < view.getPivotFields().size()) + if(view.getIncludeSummarySubTotals() && key.getKeys().size() < view.getSummaryFields().size()) { ///////////////////////////////////////////////////////////////////////////////////////// // be careful here, with these key objects, and their identity, being used as map keys // @@ -532,9 +685,9 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) + private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) throws QException { - Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); + Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); addRecordToAggregatesMap(table, record, keyAggregates); } @@ -543,23 +696,74 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) + private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) throws QException { - for(QFieldMetaData field : table.getFields().values()) + ////////////////////////////////////////////////////////////////////////////////////// + // todo - an optimization could be, to only compute aggregates that we'll need... // + // Only if we measure and see this to be slow - it may be, lots of BigDecimal math? // + ////////////////////////////////////////////////////////////////////////////////////// + for(String fieldName : record.getValues().keySet()) { - if(field.getType().equals(QFieldType.INTEGER)) + QFieldMetaData field = null; + try + { + ////////////////////////////////////////////////////// + // todo - memoize this, if we ever need to optimize // + ////////////////////////////////////////////////////// + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName); + field = fieldAndJoinTable.field(); + } + catch(Exception e) + { + ////////////////////////////////////////////////////////////////////////////////////// + // for non-real-fields... let's skip for now - but maybe treat as string in future? // + ////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Couldn't find field in table qInstance - won't compute aggregates for it", logPair("fieldName", fieldName)); + continue; + } + + if(StringUtils.hasContent(field.getPossibleValueSourceName())) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates()); - fieldAggregates.add(record.getValueInteger(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates()); + fieldAggregates.add(record.getDisplayValue(fieldName)); + } + else if(field.getType().equals(QFieldType.INTEGER)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new IntegerAggregates()); + fieldAggregates.add(record.getValueInteger(fieldName)); + } + else if(field.getType().equals(QFieldType.LONG)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LongAggregates()); + fieldAggregates.add(record.getValueLong(fieldName)); } else if(field.getType().equals(QFieldType.DECIMAL)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates()); - fieldAggregates.add(record.getValueBigDecimal(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new BigDecimalAggregates()); + fieldAggregates.add(record.getValueBigDecimal(fieldName)); + } + else if(field.getType().equals(QFieldType.DATE_TIME)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new InstantAggregates()); + fieldAggregates.add(record.getValueInstant(fieldName)); + } + else if(field.getType().equals(QFieldType.DATE)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LocalDateAggregates()); + fieldAggregates.add(record.getValueLocalDate(fieldName)); + } + if(field.getType().isStringLike()) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates()); + fieldAggregates.add(record.getValueString(fieldName)); } - // todo - more types (dates, at least?) } } @@ -568,24 +772,22 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException + private void outputSummaries(ReportInput reportInput) throws QException { - List reportViews = report.getViews().stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList(); + List reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList(); for(QReportView view : reportViews) { - QReportDataSource dataSource = report.getDataSource(view.getDataSourceName()); + QReportDataSource dataSource = getDataSource(view.getDataSourceName()); QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); SummaryOutput summaryOutput = computeSummaryRowsForView(reportInput, view, table); ExportInput exportInput = new ExportInput(); - exportInput.setReportFormat(reportFormat); - exportInput.setFilename(reportInput.getFilename()); + exportInput.setReportDestination(reportInput.getReportDestination()); exportInput.setTitleRow(summaryOutput.titleRow); exportInput.setIncludeHeaderRow(view.getIncludeHeaderRow()); - exportInput.setReportOutputStream(reportInput.getReportOutputStream()); reportStreamer.setDisplayFormats(getDisplayFormatMap(view)); - reportStreamer.start(exportInput, getFields(table, view), view.getLabel()); + reportStreamer.start(exportInput, getFields(table, view), view.getLabel(), view); reportStreamer.addRecords(summaryOutput.summaryRows); // todo - what if this set is huge? @@ -598,6 +800,24 @@ public class GenerateReportAction + /******************************************************************************* + ** + *******************************************************************************/ + private QReportDataSource getDataSource(String dataSourceName) + { + for(QReportDataSource dataSource : CollectionUtils.nonNullList(dataSources)) + { + if(dataSource.getName().equals(dataSourceName)) + { + return (dataSource); + } + } + + return (null); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -625,13 +845,13 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private List getFields(QTableMetaData table, QReportView view) + private List getFields(QTableMetaData table, QReportView view) throws QException { List fields = new ArrayList<>(); - for(String pivotField : view.getPivotFields()) + for(String summaryFieldName : view.getSummaryFields()) { - QFieldMetaData field = table.getField(pivotField); - fields.add(new QFieldMetaData(pivotField, field.getType()).withLabel(field.getLabel())); // todo do we need the type? if so need table as input here + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName); + fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type? if so need table as input here } for(QReportField column : view.getColumns()) { @@ -661,11 +881,11 @@ public class GenerateReportAction // create summary rows // ///////////////////////// List summaryRows = new ArrayList<>(); - for(Map.Entry>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet()) + for(Map.Entry>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet()) { - SummaryKey summaryKey = entry.getKey(); - Map> fieldAggregates = entry.getValue(); - Map summaryValues = getSummaryValuesForInterpreter(fieldAggregates); + SummaryKey summaryKey = entry.getKey(); + Map> fieldAggregates = entry.getValue(); + Map summaryValues = getSummaryValuesForInterpreter(fieldAggregates); variableInterpreter.addValueMap("pivot", summaryValues); variableInterpreter.addValueMap("summary", summaryValues); @@ -674,9 +894,9 @@ public class GenerateReportAction if(!varianceAggregates.isEmpty()) { - Map>> varianceMap = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap()); - Map> varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap()); - Map varianceValues = getSummaryValuesForInterpreter(varianceSubMap); + Map>> varianceMap = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap()); + Map> varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap()); + Map varianceValues = getSummaryValuesForInterpreter(varianceSubMap); variableInterpreter.addValueMap("variancePivot", varianceValues); variableInterpreter.addValueMap("variance", varianceValues); } @@ -695,7 +915,7 @@ public class GenerateReportAction /////////////////////////////////////////////////////////////////////////////// // for summary subtotals, add the text "Total" to the last field in this key // /////////////////////////////////////////////////////////////////////////////// - if(summaryKey.getKeys().size() < view.getPivotFields().size()) + if(summaryKey.getKeys().size() < view.getSummaryFields().size()) { String fieldName = summaryKey.getKeys().get(summaryKey.getKeys().size() - 1).getA(); summaryRow.setValue(fieldName, summaryRow.getValueString(fieldName) + " Total"); @@ -733,11 +953,11 @@ public class GenerateReportAction { totalRow = new QRecord(); - for(String pivotField : view.getPivotFields()) + for(String summaryField : view.getSummaryFields()) { if(totalRow.getValues().isEmpty()) { - totalRow.setValue(pivotField, "Totals"); + totalRow.setValue(summaryField, "Totals"); } } @@ -857,18 +1077,24 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private Map getSummaryValuesForInterpreter(Map> fieldAggregates) + private Map getSummaryValuesForInterpreter(Map> fieldAggregates) { Map summaryValuesForInterpreter = new HashMap<>(); - for(Map.Entry> subEntry : fieldAggregates.entrySet()) + for(Map.Entry> subEntry : fieldAggregates.entrySet()) { - String fieldName = subEntry.getKey(); - AggregatesInterface aggregates = subEntry.getValue(); + String fieldName = subEntry.getKey(); + AggregatesInterface aggregates = subEntry.getValue(); summaryValuesForInterpreter.put("sum." + fieldName, aggregates.getSum()); summaryValuesForInterpreter.put("count." + fieldName, aggregates.getCount()); + summaryValuesForInterpreter.put("count_nums." + fieldName, aggregates.getCount()); summaryValuesForInterpreter.put("min." + fieldName, aggregates.getMin()); summaryValuesForInterpreter.put("max." + fieldName, aggregates.getMax()); summaryValuesForInterpreter.put("average." + fieldName, aggregates.getAverage()); + summaryValuesForInterpreter.put("product." + fieldName, aggregates.getProduct()); + summaryValuesForInterpreter.put("var." + fieldName, aggregates.getVariance()); + summaryValuesForInterpreter.put("varp." + fieldName, aggregates.getVarP()); + summaryValuesForInterpreter.put("std_dev." + fieldName, aggregates.getStandardDeviation()); + summaryValuesForInterpreter.put("std_devp." + fieldName, aggregates.getStdDevP()); } return summaryValuesForInterpreter; } @@ -882,4 +1108,27 @@ public class GenerateReportAction { } + + + /******************************************************************************* + ** + *******************************************************************************/ + public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) + { + + /******************************************************************************* + ** + *******************************************************************************/ + public String getLabel(QTableMetaData mainTable) + { + if(mainTable.getName().equals(joinTable.getName())) + { + return (field.getLabel()); + } + else + { + return (joinTable.getLabel() + ": " + field.getLabel()); + } + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java index b90cde16..0b798fe8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java @@ -26,17 +26,23 @@ import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; /******************************************************************************* @@ -46,13 +52,23 @@ public class JsonExportStreamer implements ExportStreamerInterface { private static final QLogger LOG = QLogger.getLogger(JsonExportStreamer.class); + private boolean prettyPrint = true; + private ExportInput exportInput; private QTableMetaData table; private List fields; private OutputStream outputStream; - private boolean needComma = false; - private boolean prettyPrint = true; + private boolean multipleViews = false; + private boolean haveStartedAnyViews = false; + + private boolean needCommaBeforeRecord = false; + + private byte[] indent = new byte[0]; + private String indentString = ""; + + private Pattern colonLetterPattern = Pattern.compile(":([A-Z]+)($|[A-Z][a-z])"); + private Memoization fieldLabelMemoization = new Memoization<>(); @@ -69,21 +85,124 @@ public class JsonExportStreamer implements ExportStreamerInterface ** *******************************************************************************/ @Override - public void start(ExportInput exportInput, List fields, String label) throws QReportingException + public void preRun(ReportDestination reportDestination, List views) throws QReportingException + { + outputStream = reportDestination.getReportOutputStream(); + + if(views.size() > 1) + { + multipleViews = true; + } + + if(multipleViews) + { + try + { + indentIfPretty(outputStream); + outputStream.write('['); + newlineIfPretty(outputStream); + increaseIndent(); + } + catch(IOException e) + { + throw (new QReportingException("Error starting report output", e)); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException { this.exportInput = exportInput; this.fields = fields; table = exportInput.getTable(); - outputStream = this.exportInput.getReportOutputStream(); + + needCommaBeforeRecord = false; try { + if(multipleViews) + { + if(haveStartedAnyViews) + { + ///////////////////////// + // close the last view // + ///////////////////////// + newlineIfPretty(outputStream); + + decreaseIndent(); + indentIfPretty(outputStream); + outputStream.write(']'); + newlineIfPretty(outputStream); + + decreaseIndent(); + indentIfPretty(outputStream); + outputStream.write('}'); + outputStream.write(','); + newlineIfPretty(outputStream); + } + + ///////////////////////////////////////////////////////////// + // open a new view, as an object, with a name & data entry // + ///////////////////////////////////////////////////////////// + indentIfPretty(outputStream); + outputStream.write('{'); + newlineIfPretty(outputStream); + increaseIndent(); + + indentIfPretty(outputStream); + outputStream.write(String.format(""" + "name":"%s",""", label).getBytes(StandardCharsets.UTF_8)); + newlineIfPretty(outputStream); + + indentIfPretty(outputStream); + outputStream.write(""" + "data":""".getBytes(StandardCharsets.UTF_8)); + newlineIfPretty(outputStream); + } + + ////////////////////////////////////////////// + // start the array of entries for this view // + ////////////////////////////////////////////// + indentIfPretty(outputStream); outputStream.write('['); + increaseIndent(); } catch(IOException e) { throw (new QReportingException("Error starting report output", e)); } + + haveStartedAnyViews = true; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void increaseIndent() + { + indent = new byte[indent.length + 3]; + Arrays.fill(indent, (byte) ' '); + indentString = new String(indent); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void decreaseIndent() + { + indent = new byte[Math.max(0, indent.length - 3)]; + Arrays.fill(indent, (byte) ' '); + indentString = new String(indent); } @@ -111,7 +230,7 @@ public class JsonExportStreamer implements ExportStreamerInterface { try { - if(needComma) + if(needCommaBeforeRecord) { outputStream.write(','); } @@ -119,21 +238,25 @@ public class JsonExportStreamer implements ExportStreamerInterface Map mapForJson = new LinkedHashMap<>(); for(QFieldMetaData field : fields) { - String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", "")); - mapForJson.put(labelForJson, qRecord.getValue(field.getName())); + mapForJson.put(getLabelForJson(field), qRecord.getValue(field.getName())); } String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson); + if(prettyPrint) + { + json = json.replaceAll("(?s)\n", "\n" + indentString); + } if(prettyPrint) { outputStream.write('\n'); } + indentIfPretty(outputStream); outputStream.write(json.getBytes(StandardCharsets.UTF_8)); outputStream.flush(); // todo - less often? - needComma = true; + needCommaBeforeRecord = true; } catch(Exception e) { @@ -143,6 +266,73 @@ public class JsonExportStreamer implements ExportStreamerInterface + /******************************************************************************* + ** + *******************************************************************************/ + String getLabelForJson(QFieldMetaData field) + { + ////////////////////////////////////////////////////////////////////////// + // memoize, to avoid running these regex/replacements millions of times // + ////////////////////////////////////////////////////////////////////////// + Optional result = fieldLabelMemoization.getResult(field.getName(), fieldName -> + { + String labelForJson = field.getLabel().replace(" ", ""); + + ///////////////////////////////////////////////////////////////////////////// + // now fix up any field-name-parts after the table: portion of a join name // + // lineItem:SKU to become lineItem:sku // + // parcel:SLAStatus to become parcel:slaStatus // + // order:Client to become order:client // + ///////////////////////////////////////////////////////////////////////////// + Matcher allCaps = Pattern.compile("^[A-Z]+$").matcher(labelForJson); + Matcher startsAllCapsThenNextWordMatcher = Pattern.compile("([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson); + Matcher startsOneCapMatcher = Pattern.compile("([A-Z])(.*)").matcher(labelForJson); + + if(allCaps.matches()) + { + labelForJson = allCaps.replaceAll(m -> m.group().toLowerCase()); + } + else if(startsAllCapsThenNextWordMatcher.matches()) + { + labelForJson = startsAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2)); + } + else if(startsOneCapMatcher.matches()) + { + labelForJson = startsOneCapMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2)); + } + + ///////////////////////////////////////////////////////////////////////////// + // now fix up any field-name-parts after the table: portion of a join name // + // lineItem:SKU to become lineItem:sku // + // parcel:SLAStatus to become parcel:slaStatus // + // order:Client to become order:client // + ///////////////////////////////////////////////////////////////////////////// + Matcher colonThenAllCapsThenEndMatcher = Pattern.compile("(.*:)([A-Z]+)$").matcher(labelForJson); + Matcher colonThenAllCapsThenNextWordMatcher = Pattern.compile("(.*:)([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson); + Matcher colonThenOneCapMatcher = Pattern.compile("(.*:)([A-Z])(.*)").matcher(labelForJson); + + if(colonThenAllCapsThenEndMatcher.matches()) + { + labelForJson = colonThenAllCapsThenEndMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase()); + } + else if(colonThenAllCapsThenNextWordMatcher.matches()) + { + labelForJson = colonThenAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3)); + } + else if(colonThenOneCapMatcher.matches()) + { + labelForJson = colonThenOneCapMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3)); + } + + System.out.println("Label: " + labelForJson); + return (labelForJson); + }); + + return result.orElse(field.getName()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -162,11 +352,34 @@ public class JsonExportStreamer implements ExportStreamerInterface { try { - if(prettyPrint) - { - outputStream.write('\n'); - } + ////////////////////////////////////////////// + // close the array of entries for this view // + ////////////////////////////////////////////// + newlineIfPretty(outputStream); + + decreaseIndent(); + indentIfPretty(outputStream); outputStream.write(']'); + newlineIfPretty(outputStream); + + if(multipleViews) + { + //////////////////////////////////////////// + // close this view, if there are multiple // + //////////////////////////////////////////// + decreaseIndent(); + indentIfPretty(outputStream); + outputStream.write('}'); + newlineIfPretty(outputStream); + + ///////////////////////////// + // close the list of views // + ///////////////////////////// + decreaseIndent(); + indentIfPretty(outputStream); + outputStream.write(']'); + newlineIfPretty(outputStream); + } } catch(IOException e) { @@ -174,4 +387,30 @@ public class JsonExportStreamer implements ExportStreamerInterface } } + + + /******************************************************************************* + ** + *******************************************************************************/ + private void newlineIfPretty(OutputStream outputStream) throws IOException + { + if(prettyPrint) + { + outputStream.write('\n'); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void indentIfPretty(OutputStream outputStream) throws IOException + { + if(prettyPrint) + { + outputStream.write(indent); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java index 365c3a40..7e422a21 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; /******************************************************************************* @@ -87,7 +88,7 @@ public class ListOfMapsExportStreamer implements ExportStreamerInterface ** *******************************************************************************/ @Override - public void start(ExportInput exportInput, List fields, String label) throws QReportingException + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException { this.exportInput = exportInput; this.fields = fields; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index 2583855f..4964f221 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -43,8 +44,9 @@ public class RecordPipe private static final long BLOCKING_SLEEP_MILLIS = 100; private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes + private static final int DEFAULT_CAPACITY = 1_000; - private int capacity = 1_000; + private int capacity = DEFAULT_CAPACITY; private ArrayBlockingQueue queue = new ArrayBlockingQueue<>(capacity); private boolean isTerminated = false; @@ -70,11 +72,13 @@ public class RecordPipe /******************************************************************************* ** Construct a record pipe, with an alternative capacity for the internal queue. + ** + ** overrideCapacity is allowed to be null - in which case, DEFAULT_CAPACITY is used. *******************************************************************************/ public RecordPipe(Integer overrideCapacity) { - this.capacity = overrideCapacity; - queue = new ArrayBlockingQueue<>(overrideCapacity); + this.capacity = Objects.requireNonNullElse(overrideCapacity, DEFAULT_CAPACITY); + queue = new ArrayBlockingQueue<>(this.capacity); } @@ -138,7 +142,7 @@ public class RecordPipe { if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS) { - LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS); + LOG.warn("Giving up adding record to pipe, due to pipe being full for more than " + MAX_SLEEP_LOOP_MILLIS + " millis"); throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long.")); } LOG.trace("Record pipe.add failed (due to full pipe). Blocking."); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java new file mode 100644 index 00000000..9c682e22 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ReportUtils +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static QReportView getSourceViewForPivotTableView(List views, QReportView pivotTableView) throws QReportingException + { + Optional sourceView = views.stream().filter(v -> v.getName().equals(pivotTableView.getPivotTableSourceViewName())).findFirst(); + if(sourceView.isEmpty()) + { + throw (new QReportingException("Could not find data view [" + pivotTableView.getPivotTableSourceViewName() + "] for pivot table view [" + pivotTableView.getName() + "]")); + } + + return sourceView.get(); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/BoldHeaderAndFooterExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/BoldHeaderAndFooterFastExcelStyler.java similarity index 85% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/BoldHeaderAndFooterExcelStyler.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/BoldHeaderAndFooterFastExcelStyler.java index 12dc6685..45d37640 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/BoldHeaderAndFooterExcelStyler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/BoldHeaderAndFooterFastExcelStyler.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting; +package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel; import org.dhatim.fastexcel.BorderSide; @@ -30,7 +30,7 @@ import org.dhatim.fastexcel.StyleSetter; /******************************************************************************* ** Version of excel styler that does bold headers and footers, with basic borders. *******************************************************************************/ -public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface +public class BoldHeaderAndFooterFastExcelStyler implements FastExcelStylerInterface { /******************************************************************************* @@ -60,6 +60,9 @@ public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface + /******************************************************************************* + ** + *******************************************************************************/ @Override public void styleTotalsRow(StyleSetter totalsRowStyle) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java similarity index 89% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java index 61f14588..fd1dc915 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.reporting; +package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel; import java.io.OutputStream; @@ -34,8 +34,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.ExcelStylerInterface; -import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.PlainExcelStyler; +import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; @@ -43,7 +43,9 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.dhatim.fastexcel.StyleSetter; @@ -52,19 +54,19 @@ import org.dhatim.fastexcel.Worksheet; /******************************************************************************* - ** Excel export format implementation + ** Excel export format implementation - built using fastexcel library *******************************************************************************/ -public class ExcelExportStreamer implements ExportStreamerInterface +public class ExcelFastexcelExportStreamer implements ExportStreamerInterface { - private static final QLogger LOG = QLogger.getLogger(ExcelExportStreamer.class); + private static final QLogger LOG = QLogger.getLogger(ExcelFastexcelExportStreamer.class); private ExportInput exportInput; private QTableMetaData table; private List fields; private OutputStream outputStream; - private ExcelStylerInterface excelStylerInterface = new PlainExcelStyler(); - private Map excelCellFormats; + private FastExcelStylerInterface fastExcelStylerInterface = new PlainFastExcelStyler(); + private Map excelCellFormats; private Workbook workbook; private Worksheet worksheet; @@ -76,7 +78,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface /******************************************************************************* ** *******************************************************************************/ - public ExcelExportStreamer() + public ExcelFastexcelExportStreamer() { } @@ -105,14 +107,14 @@ public class ExcelExportStreamer implements ExportStreamerInterface ** Starts a new worksheet in the current workbook. Can be called multiple times. *******************************************************************************/ @Override - public void start(ExportInput exportInput, List fields, String label) throws QReportingException + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException { try { this.exportInput = exportInput; this.fields = fields; table = exportInput.getTable(); - outputStream = this.exportInput.getReportOutputStream(); + outputStream = this.exportInput.getReportDestination().getReportOutputStream(); this.row = 0; this.sheetCount++; @@ -121,7 +123,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface ///////////////////////////////////////////////////////////////////////////////////////////////////// if(workbook == null) { - String appName = "QQQ"; + String appName = ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getBranding().getAppName(), "QQQ"); QInstance instance = exportInput.getInstance(); if(instance != null && instance.getBranding() != null && instance.getBranding().getCompanyName() != null) { @@ -167,7 +169,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface worksheet.range(row, 0, row, fields.size() - 1).merge(); StyleSetter titleStyle = worksheet.range(row, 0, row, fields.size() - 1).style(); - excelStylerInterface.styleTitleRow(titleStyle); + fastExcelStylerInterface.styleTitleRow(titleStyle); titleStyle.set(); row++; @@ -187,7 +189,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface } StyleSetter headerStyle = worksheet.range(row, 0, row, fields.size() - 1).style(); - excelStylerInterface.styleHeaderRow(headerStyle); + fastExcelStylerInterface.styleHeaderRow(headerStyle); headerStyle.set(); row++; @@ -315,7 +317,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface writeRecord(record); StyleSetter totalsRowStyle = worksheet.range(row, 0, row, fields.size() - 1).style(); - excelStylerInterface.styleTotalsRow(totalsRowStyle); + fastExcelStylerInterface.styleTotalsRow(totalsRowStyle); totalsRowStyle.set(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/ExcelStylerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/FastExcelStylerInterface.java similarity index 92% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/ExcelStylerInterface.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/FastExcelStylerInterface.java index ed58aba3..68b361eb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/ExcelStylerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/FastExcelStylerInterface.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting; +package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel; import org.dhatim.fastexcel.StyleSetter; @@ -29,7 +29,7 @@ import org.dhatim.fastexcel.StyleSetter; ** Interface for classes that know how to apply styles to an Excel stream being ** built by fastexcel. *******************************************************************************/ -public interface ExcelStylerInterface +public interface FastExcelStylerInterface { /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java new file mode 100644 index 00000000..e42afe36 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java @@ -0,0 +1,43 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel; + + +import org.dhatim.fastexcel.StyleSetter; + + +/******************************************************************************* + ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface. + *******************************************************************************/ +public class PlainFastExcelStyler implements FastExcelStylerInterface +{ + + /******************************************************************************* + ** ... sorry, but adding this gives us test coverage on this class, even though + ** we're just deferring to super... + *******************************************************************************/ + @Override + public void styleHeaderRow(StyleSetter headerRowStyle) + { + FastExcelStylerInterface.super.styleHeaderRow(headerRowStyle); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java new file mode 100644 index 00000000..c30beef1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java @@ -0,0 +1,93 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import org.apache.poi.ss.usermodel.BorderStyle; +import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + + +/******************************************************************************* + ** Version of POI excel styler that does bold headers and footers, with basic borders. + *******************************************************************************/ +public class BoldHeaderAndFooterPoiExcelStyler implements PoiExcelStylerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper) + { + Font font = workbook.createFont(); + font.setFontHeightInPoints((short) 14); + font.setBold(true); + + XSSFCellStyle cellStyle = workbook.createCellStyle(); + cellStyle.setFont(font); + cellStyle.setAlignment(HorizontalAlignment.CENTER); + + return (cellStyle); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper) + { + Font font = workbook.createFont(); + font.setBold(true); + + XSSFCellStyle cellStyle = workbook.createCellStyle(); + cellStyle.setFont(font); + cellStyle.setBorderBottom(BorderStyle.THIN); + + return (cellStyle); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper) + { + Font font = workbook.createFont(); + font.setBold(true); + + XSSFCellStyle cellStyle = workbook.createCellStyle(); + cellStyle.setFont(font); + cellStyle.setBorderTop(BorderStyle.THIN); + cellStyle.setBorderBottom(BorderStyle.DOUBLE); + + return (cellStyle); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java new file mode 100644 index 00000000..1ce29911 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java @@ -0,0 +1,819 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Serializable; +import java.io.Writer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; +import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; +import com.kingsrook.qqq.backend.core.actions.reporting.ReportUtils; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.poi.ss.SpreadsheetVersion; +import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.ss.usermodel.DataConsolidateFunction; +import org.apache.poi.ss.usermodel.DateUtil; +import org.apache.poi.ss.util.AreaReference; +import org.apache.poi.ss.util.CellReference; +import org.apache.poi.ss.util.WorkbookUtil; +import org.apache.poi.xssf.usermodel.XSSFCell; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFPivotTable; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + + +/******************************************************************************* + ** Excel export format implementation using POI library, but with modifications + ** to actually stream output rather than use any temp files. + ** + ** For a rough outline: + ** - create a basically empty Excel workbook using POI - empty meaning, without + ** data rows. + ** - have POI write that workbook out into a byte[] - which will be a zip full + ** of xml (e.g., xlsx). + ** - then open a new ZipOutputStream wrapper around the OutputStream we took in + ** as the report destination (e.g., streamed into storage or http output) + ** - Copy over all entries from the xlsx into our new zip-output-stream, other than + ** ones that are the actual sheets that we want to put data into. + ** - For the sheet entries, use the StreamedPoiSheetWriter class to write the + ** report's data directly out as Excel XML (not using POI). + ** - Pivot tables require a bit of an additional hack - to write a "pivot cache + ** definition", which, while we won't put all the data in it (because we'll tell + ** it refreshOnLoad="true"), we will at least need to know how many cols & rows + ** are in the data-sheet (which we wouldn't know until we streamed that sheet!) + *******************************************************************************/ +public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInterface +{ + private static final QLogger LOG = QLogger.getLogger(ExcelPoiBasedStreamingExportStreamer.class); + + private List views; + private ExportInput exportInput; + private List fields; + private OutputStream outputStream; + private ZipOutputStream zipOutputStream; + + public static final String EXCEL_DATE_FORMAT = "yyyy-MM-dd"; + public static final String EXCEL_DATE_TIME_FORMAT = "yyyy-MM-dd H:mm:ss"; + + private PoiExcelStylerInterface poiExcelStylerInterface = getStylerInterface(); + private Map excelCellFormats; + private Map styles = new HashMap<>(); + + private int rowNo = 0; + private int sheetIndex = 1; + + private Map pivotViewToCacheDefinitionReferenceMap = new HashMap<>(); + + private Writer activeSheetWriter = null; + private StreamedSheetWriter sheetWriter = null; + + private QReportView currentView = null; + private Map> fieldsPerView = new HashMap<>(); + private Map rowsPerView = new HashMap<>(); + private Map labelViewsByName = new HashMap<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ExcelPoiBasedStreamingExportStreamer() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(ReportDestination reportDestination, List views) throws QReportingException + { + try + { + this.outputStream = reportDestination.getReportOutputStream(); + this.views = views; + + /////////////////////////////////////////////////////////////////////////////// + // create 'template' workbook through poi - with sheets corresponding to our // + // actual file this will be a zip file (stream), with entries for all of the // + // files in the final xlsx but without any data, so it'll be small // + /////////////////////////////////////////////////////////////////////////////// + XSSFWorkbook workbook = new XSSFWorkbook(); + createStyles(workbook); + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // for each of the sheets, create it in the workbook, and put a reference to it in the sheetMap // + ////////////////////////////////////////////////////////////////////////////////////////////////// + Map sheetMapByExcelReference = new HashMap<>(); + Map sheetMapByViewName = new HashMap<>(); + + int sheetCounter = 1; + for(QReportView view : views) + { + String label = Objects.requireNonNullElse(view.getLabel(), "Sheet " + sheetCounter); + label = WorkbookUtil.createSafeSheetName(label); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // track the actually-used sheet labels (needed for referencing in pivot table generation) // + ///////////////////////////////////////////////////////////////////////////////////////////// + labelViewsByName.put(view.getName(), label); + + XSSFSheet sheet = workbook.createSheet(label); + String sheetReference = sheet.getPackagePart().getPartName().getName().substring(1); + sheetMapByExcelReference.put(sheetReference, sheet); + sheetMapByViewName.put(view.getName(), sheet); + sheetCounter++; + } + + //////////////////////////////////////////////////// + // if any views are pivot tables, create them now // + //////////////////////////////////////////////////// + List pivotViewNames = new ArrayList<>(); + for(QReportView view : views) + { + if(ReportType.PIVOT.equals(view.getType())) + { + pivotViewNames.add(view.getName()); + + XSSFSheet pivotTableSheet = Objects.requireNonNull(sheetMapByViewName.get(view.getName()), "Could not get pivot table sheet view by name: " + view.getName()); + XSSFSheet dataSheet = Objects.requireNonNull(sheetMapByViewName.get(view.getPivotTableSourceViewName()), "Could not get pivot table source sheet by view name: " + view.getPivotTableSourceViewName()); + QReportView dataView = ReportUtils.getSourceViewForPivotTableView(views, view); + createPivotTableTemplate(pivotTableSheet, view, dataSheet, dataView); + } + } + Iterator pivotViewNameIterator = pivotViewNames.iterator(); + + ///////////////////////////////////////////////////////// + // write that template worksheet zip out to byte array // + ///////////////////////////////////////////////////////// + ByteArrayOutputStream templateBAOS = new ByteArrayOutputStream(); + workbook.write(templateBAOS); + templateBAOS.close(); + byte[] templateBytes = templateBAOS.toByteArray(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // open up a zipOutputStream around the output stream that the report is to be written to. // + // note: this stream is closed in the finish method - see more comments there. // + ///////////////////////////////////////////////////////////////////////////////////////////// + this.zipOutputStream = new ZipOutputStream(this.outputStream); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // copy over all the entries in the template zip that aren't the sheets into the output stream // + ///////////////////////////////////////////////////////////////////////////////////////////////// + ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(templateBytes)); + ZipEntry zipTemplateEntry = null; + byte[] buffer = new byte[2048]; + while((zipTemplateEntry = zipInputStream.getNextEntry()) != null) + { + if(zipTemplateEntry.getName().matches(".*/pivotCacheDefinition.*.xml")) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this zip entry is a pivotCacheDefinition, then don't write it to the output stream right now. // + // instead, just map the pivot view's name to the zipTemplateEntry name // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!pivotViewNameIterator.hasNext()) + { + throw new QReportingException("Found a pivot cache definition [" + zipTemplateEntry.getName() + "] in the template ZIP, but no (more) corresponding pivot view names"); + } + + String pivotViewName = pivotViewNameIterator.next(); + LOG.info("Holding on a pivot cache definition zip template entry [" + pivotViewName + "] [" + zipTemplateEntry.getName() + "]..."); + pivotViewToCacheDefinitionReferenceMap.put(pivotViewName, zipTemplateEntry.getName()); + } + else if(!sheetMapByExcelReference.containsKey(zipTemplateEntry.getName())) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, if we don't have this zipTemplateEntry name in our map of sheets, then this is a kinda "meta" // + // file that we don't really care about (e.g., not our sheet data), so just copy it to the output stream. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.info("Copying zip template entry [" + zipTemplateEntry.getName() + "] to output stream"); + zipOutputStream.putNextEntry(new ZipEntry(zipTemplateEntry.getName())); + + int length; + while((length = zipInputStream.read(buffer)) > 0) + { + zipOutputStream.write(buffer, 0, length); + } + + zipInputStream.closeEntry(); + } + else + { + //////////////////////////////////////////////////////////////////////////////////// + // else - this is a sheet - so again, don't write it yet - stream its data below. // + //////////////////////////////////////////////////////////////////////////////////// + LOG.info("Skipping presumed sheet zip template entry [" + zipTemplateEntry.getName() + "] to output stream"); + } + } + + zipInputStream.close(); + } + catch(Exception e) + { + throw (new QReportingException("Error preparing to generate spreadsheet", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void createPivotTableTemplate(XSSFSheet pivotTableSheet, QReportView pivotView, XSSFSheet dataSheet, QReportView dataView) throws QReportingException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // write just enough data to the dataView's sheet so that we can refer to it for creating the pivot table. // + // we need to do this, because POI will try to create the pivotCache referring to the data sheet, and if // + // there isn't any data there, it'll crash. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + XSSFRow headerRow = dataSheet.createRow(0); + int columnNo = 0; + for(QReportField column : dataView.getColumns()) + { + XSSFCell cell = headerRow.createCell(columnNo++); + cell.setCellValue(QInstanceEnricher.nameToLabel(column.getName())); + } + + XSSFRow valuesRow = dataSheet.createRow(1); + columnNo = 0; + for(QReportField column : dataView.getColumns()) + { + XSSFCell cell = valuesRow.createCell(columnNo++); + cell.setCellValue("Value " + columnNo); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // for this template version of the pivot table, tell it there are only 2 rows in the source sheet // + // as that's all that we wrote above (a header and 1 fake value row) // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + int rows = 2; + String colsLetter = CellReference.convertNumToColString(dataView.getColumns().size() - 1); + AreaReference source = new AreaReference("A1:" + colsLetter + rows, SpreadsheetVersion.EXCEL2007); + CellReference position = new CellReference("A1"); + + ////////////////////////////////////////////////////////////////// + // tell poi all about our pivot table - rows, cols, and columns // + ////////////////////////////////////////////////////////////////// + XSSFPivotTable pivotTable = pivotTableSheet.createPivotTable(source, position, dataSheet); + + for(PivotTableGroupBy row : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getRows())) + { + int rowLabelColumnIndex = getColumnIndex(dataView.getColumns(), row.getFieldName()); + pivotTable.addRowLabel(rowLabelColumnIndex); + } + + for(PivotTableGroupBy column : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getColumns())) + { + int colLabelColumnIndex = getColumnIndex(dataView.getColumns(), column.getFieldName()); + pivotTable.addColLabel(colLabelColumnIndex); + } + + for(PivotTableValue value : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getValues())) + { + int columnLabelColumnIndex = getColumnIndex(dataView.getColumns(), value.getFieldName()); + + String labelPrefix = value.getFunction().name() + " of "; + String label = labelPrefix + QInstanceEnricher.nameToLabel(value.getFieldName()); + String valueFormat = null; + + Optional optSourceField = dataView.getColumns().stream().filter(c -> c.getName().equals(value.getFieldName())).findFirst(); + if(optSourceField.isPresent()) + { + QReportField sourceField = optSourceField.get(); + + if(StringUtils.hasContent(sourceField.getLabel())) + { + label = labelPrefix + sourceField.getLabel(); + } + + if(StringUtils.hasContent(sourceField.getDisplayFormat())) + { + valueFormat = DisplayFormat.getExcelFormat(sourceField.getDisplayFormat()); + } + else + { + if(QFieldType.DATE.equals(sourceField.getType())) + { + valueFormat = EXCEL_DATE_FORMAT; + } + else if(QFieldType.DATE_TIME.equals(sourceField.getType())) + { + valueFormat = EXCEL_DATE_TIME_FORMAT; + } + } + } + + pivotTable.addColumnLabel(DataConsolidateFunction.valueOf(value.getFunction().name()), columnLabelColumnIndex, label, valueFormat); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private int getColumnIndex(List columns, String fieldName) throws QReportingException + { + for(int i = 0; i < columns.size(); i++) + { + if(columns.get(i).getName().equals(fieldName)) + { + return (i); + } + } + + throw (new QReportingException("Could not find column by name [" + fieldName + "]")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void createStyles(XSSFWorkbook workbook) + { + CreationHelper createHelper = workbook.getCreationHelper(); + + XSSFCellStyle dateStyle = workbook.createCellStyle(); + dateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT)); + styles.put("date", dateStyle); + + XSSFCellStyle dateTimeStyle = workbook.createCellStyle(); + dateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT)); + styles.put("datetime", dateTimeStyle); + + styles.put("title", poiExcelStylerInterface.createStyleForTitle(workbook, createHelper)); + styles.put("header", poiExcelStylerInterface.createStyleForHeader(workbook, createHelper)); + styles.put("footer", poiExcelStylerInterface.createStyleForFooter(workbook, createHelper)); + + XSSFCellStyle footerDateStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper); + footerDateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT)); + styles.put("footer-date", footerDateStyle); + + XSSFCellStyle footerDateTimeStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper); + footerDateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT)); + styles.put("footer-datetime", footerDateTimeStyle); + } + + + + /******************************************************************************* + ** Starts a new worksheet in the current workbook. Can be called multiple times. + *******************************************************************************/ + @Override + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException + { + try + { + ///////////////////////////////////////// + // close previous sheet if one is open // + ///////////////////////////////////////// + closeLastSheetIfOpen(); + + if(currentView != null) + { + this.rowsPerView.put(currentView.getName(), rowNo); + } + + this.currentView = view; + this.exportInput = exportInput; + this.fields = fields; + this.rowNo = 0; + + this.fieldsPerView.put(view.getName(), fields); + + ////////////////////////////////////////// + // start the new sheet as: // + // - a new entry in the zipOutputStream // + // - with a new output stream writer // + // - and with a SpreadsheetWriter // + ////////////////////////////////////////// + zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml")); + activeSheetWriter = new OutputStreamWriter(zipOutputStream); + sheetWriter = new StreamedSheetWriter(activeSheetWriter); + + if(ReportType.PIVOT.equals(view.getType())) + { + writePivotTable(view, ReportUtils.getSourceViewForPivotTableView(views, view)); + } + else + { + sheetWriter.beginSheet(); + + //////////////////////////////////////////////// + // put the title and header rows in the sheet // + //////////////////////////////////////////////// + writeTitleAndHeader(); + } + } + catch(Exception e) + { + throw (new QReportingException("Error starting worksheet", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTitleAndHeader() throws QReportingException + { + try + { + /////////////// + // title row // + /////////////// + if(StringUtils.hasContent(exportInput.getTitleRow())) + { + sheetWriter.insertRow(rowNo++); + sheetWriter.createCell(0, exportInput.getTitleRow(), styles.get("title").getIndex()); + sheetWriter.endRow(); + } + + //////////////// + // header row // + //////////////// + if(exportInput.getIncludeHeaderRow()) + { + sheetWriter.insertRow(rowNo++); + XSSFCellStyle headerStyle = styles.get("header"); + + int col = 0; + for(QFieldMetaData column : fields) + { + sheetWriter.createCell(col, column.getLabel(), headerStyle.getIndex()); + col++; + } + + sheetWriter.endRow(); + } + } + catch(Exception e) + { + throw (new QReportingException("Error starting Excel report")); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addRecords(List qRecords) throws QReportingException + { + LOG.info("Consuming [" + qRecords.size() + "] records from the pipe"); + + try + { + for(QRecord qRecord : qRecords) + { + writeRecord(qRecord); + } + } + catch(Exception e) + { + LOG.error("Exception generating excel file", e); + try + { + outputStream.close(); + } + catch(IOException ex) + { + LOG.warn("Secondary error closing excel output stream", e); + } + + throw (new QReportingException("Error generating Excel report", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeRecord(QRecord qRecord) throws IOException + { + writeRecord(qRecord, false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeRecord(QRecord qRecord, boolean isFooter) throws IOException + { + sheetWriter.insertRow(rowNo++); + + int styleIndex = -1; + int dateStyleIndex = styles.get("date").getIndex(); + int dateTimeStyleIndex = styles.get("datetime").getIndex(); + if(isFooter) + { + styleIndex = styles.get("footer").getIndex(); + dateStyleIndex = styles.get("footer-date").getIndex(); + dateTimeStyleIndex = styles.get("footer-datetime").getIndex(); + } + + int col = 0; + for(QFieldMetaData field : fields) + { + Serializable value = qRecord.getValue(field.getName()); + + if(value != null) + { + if(value instanceof String s) + { + sheetWriter.createCell(col, s, styleIndex); + } + else if(value instanceof Number n) + { + sheetWriter.createCell(col, n.doubleValue(), styleIndex); + + if(excelCellFormats != null) + { + String format = excelCellFormats.get(field.getName()); + if(format != null) + { + //////////////////////////////////////////////////////////////////////////////////////////// + // todo - so - for this streamed/zip approach, we need to know all styles before we start // + // any sheets. but, right now Report action only calls us with per-sheet styles when // + // it's starting individual sheets. so, we can't quite support this at this time. // + // "just" need to change GenerateReportAction to look up all cell formats for all sheets // + // before preRun is called... and change all existing streamer classes to handle that too // + //////////////////////////////////////////////////////////////////////////////////////////// + // worksheet.style(rowNo, col).format(format).set(); + } + } + } + else if(value instanceof Boolean b) + { + sheetWriter.createCell(col, b, styleIndex); + } + else if(value instanceof Date d) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex); + } + else if(value instanceof LocalDate d) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex); + } + else if(value instanceof LocalDateTime d) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex); + } + else if(value instanceof ZonedDateTime d) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(d.toLocalDateTime()), dateTimeStyleIndex); + } + else if(value instanceof Instant i) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(i.atZone(ZoneId.systemDefault()).toLocalDateTime()), dateTimeStyleIndex); + } + else + { + sheetWriter.createCell(col, ValueUtils.getValueAsString(value), styleIndex); + } + } + + col++; + } + + sheetWriter.endRow(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addTotalsRow(QRecord record) throws QReportingException + { + try + { + writeRecord(record, true); + } + catch(Exception e) + { + throw (new QReportingException("Error adding totals row", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void finish() throws QReportingException + { + try + { + ////////////////////////////////////////////// + // close the last open sheet if one is open // + ////////////////////////////////////////////// + closeLastSheetIfOpen(); + + ///////////////////////////////////////////////////////////////////////////////////// + // so, we DO need a close here, on the zipOutputStream, to finish its "zippiness". // + // even though, doing so also closes the outputStream from the caller that this // + // zipOutputStream is wrapping (and the caller will likely call close on too)... // + ///////////////////////////////////////////////////////////////////////////////////// + zipOutputStream.close(); + } + catch(Exception e) + { + throw (new QReportingException("Error finishing Excel report", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void closeLastSheetIfOpen() throws IOException + { + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we have an active sheet writer: // + // - end the current sheet in the spreadsheet writer (write some closing xml, unless it's a pivot!) // + // - flush the contents through the activeSheetWriter // + // - close the zip entry in the output stream. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + if(activeSheetWriter != null) + { + if(!ReportType.PIVOT.equals(currentView.getType())) + { + sheetWriter.endSheet(); + } + + activeSheetWriter.flush(); + zipOutputStream.closeEntry(); + } + } + + + + /******************************************************************************* + ** display formats is a map of field name to Excel format strings (e.g., $#,##0.00) + *******************************************************************************/ + @Override + public void setDisplayFormats(Map displayFormats) + { + this.excelCellFormats = new HashMap<>(); + for(Map.Entry entry : displayFormats.entrySet()) + { + String excelFormat = DisplayFormat.getExcelFormat(entry.getValue()); + if(excelFormat != null) + { + excelCellFormats.put(entry.getKey(), excelFormat); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writePivotTable(QReportView pivotTableView, QReportView dataView) throws QReportingException + { + try + { + ////////////////////////////////////////////////////////////////////////////////// + // write the xml file that is the pivot table sheet. // + // note that the ZipEntry here will have been started above in the start method // + ////////////////////////////////////////////////////////////////////////////////// + activeSheetWriter.write(""" + + + + + + + + + + + + + + """); + activeSheetWriter.flush(); + + //////////////////////////////////////////////////////////////////////////// + // start a new zip entry, for this pivot view's cacheDefinition reference // + //////////////////////////////////////////////////////////////////////////// + zipOutputStream.putNextEntry(new ZipEntry(pivotViewToCacheDefinitionReferenceMap.get(pivotTableView.getName()))); + + ///////////////////////////////////////////////////////// + // prepare the xml for each field (e.g., w/ its label) // + ///////////////////////////////////////////////////////// + List cachedFieldElements = new ArrayList<>(); + for(QFieldMetaData column : this.fieldsPerView.get(dataView.getName())) + { + cachedFieldElements.add(String.format(""" + + + + """, column.getLabel())); + } + + ///////////////////////////////////////////////////////////////////////////////////// + // write the xml file that is the pivot cache definition (structure only, no data) // + ///////////////////////////////////////////////////////////////////////////////////// + activeSheetWriter = new OutputStreamWriter(zipOutputStream); + activeSheetWriter.write(String.format(""" + + + + + + + %s + + + """, + StreamedSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())), + CellReference.convertNumToColString(dataView.getColumns().size() - 1), + rowsPerView.get(dataView.getName()), + dataView.getColumns().size(), + StringUtils.join("\n", cachedFieldElements))); + } + catch(Exception e) + { + throw (new QReportingException("Error writing pivot table", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected PoiExcelStylerInterface getStylerInterface() + { + return (new PlainPoiExcelStyler()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java new file mode 100644 index 00000000..15f43ee0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + + +/******************************************************************************* + ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface. + *******************************************************************************/ +public class PlainPoiExcelStyler implements PoiExcelStylerInterface +{ + + /******************************************************************************* + ** ... sorry, but adding this gives us test coverage on this class, even though + ** we're just deferring to super... + *******************************************************************************/ + @Override + public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper) + { + return PoiExcelStylerInterface.super.createStyleForHeader(workbook, createHelper); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java new file mode 100644 index 00000000..c052e194 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + + +/******************************************************************************* + ** Interface for classes that know how to apply styles to an Excel stream being + ** built by POI. + *******************************************************************************/ +public interface PoiExcelStylerInterface +{ + /******************************************************************************* + ** + *******************************************************************************/ + default XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper) + { + return (workbook.createCellStyle()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + default XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper) + { + return (workbook.createCellStyle()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + default XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper) + { + return (workbook.createCellStyle()); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java new file mode 100644 index 00000000..850a1ecc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java @@ -0,0 +1,242 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import java.io.IOException; +import java.io.Writer; +import java.util.HashMap; +import java.util.Map; +import org.apache.poi.ss.util.CellReference; + + +/******************************************************************************* + ** Write excel formatted XML to a Writer. + ** Originally from https://coderanch.com/t/548897/java/Generate-large-excel-POI + *******************************************************************************/ +public class StreamedSheetWriter +{ + private final Writer writer; + private int rowNo; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public StreamedSheetWriter(Writer writer) + { + this.writer = writer; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void beginSheet() throws IOException + { + writer.write(""" + + + """); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void endSheet() throws IOException + { + writer.write(""" + + """); + } + + + + /******************************************************************************* + ** Insert a new row + ** + ** @param rowNo 0-based row number + *******************************************************************************/ + public void insertRow(int rowNo) throws IOException + { + writer.write("\n"); + this.rowNo = rowNo; + } + + + + /******************************************************************************* + ** Insert row end marker + *******************************************************************************/ + public void endRow() throws IOException + { + writer.write("\n"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, String value, int styleIndex) throws IOException + { + String ref = new CellReference(rowNo, columnIndex).formatAsString(); + writer.write(""); + writer.write("" + cleanValue + ""); + writer.write(""); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String cleanseValue(String value) + { + if(value != null) + { + StringBuilder rs = new StringBuilder(); + for(int i = 0; i < value.length(); i++) + { + char c = value.charAt(i); + if(c == '&') + { + rs.append("&"); + } + else if(c == '<') + { + rs.append("<"); + } + else if(c == '>') + { + rs.append(">"); + } + else if(c == '\'') + { + rs.append("'"); + } + else if(c == '"') + { + rs.append("""); + } + else if (c < 32 && c != '\t' && c != '\n') + { + rs.append(' '); + } + else + { + rs.append(c); + } + } + + Map m = new HashMap(); + m.computeIfAbsent("s", (s) -> 3); + + value = rs.toString(); + } + + return (value); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, String value) throws IOException + { + createCell(columnIndex, value, -1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, double value, int styleIndex) throws IOException + { + String ref = new CellReference(rowNo, columnIndex).formatAsString(); + writer.write(""); + writer.write("" + value + ""); + writer.write(""); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, double value) throws IOException + { + createCell(columnIndex, value, -1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, Boolean value) throws IOException + { + createCell(columnIndex, value, -1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, Boolean value, int styleIndex) throws IOException + { + String ref = new CellReference(rowNo, columnIndex).formatAsString(); + writer.write(""); + if(value != null) + { + writer.write("" + (value ? 1 : 0) + ""); + } + writer.write(""); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java index 2e7e811f..5aa0be68 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java @@ -77,7 +77,6 @@ public class ExecuteCodeAction /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") public void run(ExecuteCodeInput input, ExecuteCodeOutput output) throws QException, QCodeException { QCodeReference codeReference = input.getCodeReference(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index ee49499a..f964b62e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -35,9 +35,8 @@ import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostDeleteCustomizer; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper; @@ -250,7 +249,7 @@ public class DeleteAction ////////////////////////////////////////////////////////////// // finally, run the post-delete customizer, if there is one // ////////////////////////////////////////////////////////////// - Optional postDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPostDeleteCustomizer.class, table, TableCustomizers.POST_DELETE_RECORD.getRole()); + Optional postDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_DELETE_RECORD.getRole()); if(postDeleteCustomizer.isPresent() && oldRecordList.isPresent()) { //////////////////////////////////////////////////////////////////////////// @@ -260,8 +259,7 @@ public class DeleteAction try { - postDeleteCustomizer.get().setDeleteInput(deleteInput); - List postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer); + List postCustomizerResult = postDeleteCustomizer.get().postDelete(deleteInput, recordsForCustomizer); /////////////////////////////////////////////////////// // check if any records got errors in the customizer // @@ -327,13 +325,11 @@ public class DeleteAction /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-delete customizer, if there is one // /////////////////////////////////////////////////////////////////////////// - Optional preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole()); + Optional preDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_DELETE_RECORD.getRole()); List customizerResult = oldRecordList.get(); if(preDeleteCustomizer.isPresent()) { - preDeleteCustomizer.get().setDeleteInput(deleteInput); - preDeleteCustomizer.get().setIsPreview(isPreview); - customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get()); + customizerResult = preDeleteCustomizer.get().preDelete(deleteInput, oldRecordList.get(), isPreview); } ///////////////////////////////////////////////////////////////////////// @@ -474,7 +470,7 @@ public class DeleteAction QRecord recordWithError = new QRecord(); recordsWithErrors.add(recordWithError); recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue); - recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue)); + recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + Objects.requireNonNullElse(primaryKeyField.getLabel(), primaryKeyField.getName()) + " = " + primaryKeyValue)); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 728f5874..e2524591 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -27,8 +27,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.ActionHelper; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.GetActionCacheHelper; @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; /******************************************************************************* @@ -57,23 +58,13 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class GetAction { - private Optional postGetRecordCustomizer; + private Optional postGetRecordCustomizer; private GetInput getInput; private QPossibleValueTranslator qPossibleValueTranslator; - /******************************************************************************* - ** - *******************************************************************************/ - public QRecord executeForRecord(GetInput getInput) throws QException - { - return (execute(getInput).getRecord()); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -87,7 +78,7 @@ public class GetAction throw (new QException("Requested to Get a record from an unrecognized table: " + getInput.getTableName())); } - postGetRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole()); + postGetRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole()); this.getInput = getInput; QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); @@ -107,9 +98,11 @@ public class GetAction } GetOutput getOutput; + boolean usingDefaultGetInterface = false; if(getInterface == null) { getInterface = new DefaultGetInterface(); + usingDefaultGetInterface = true; } getInterface.validateInput(getInput); @@ -123,10 +116,11 @@ public class GetAction new GetActionCacheHelper().handleCaching(getInput, getOutput); } - //////////////////////////////////////////////////////// - // if the record is found, perform post-actions on it // - //////////////////////////////////////////////////////// - if(getOutput.getRecord() != null) + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the record is found, perform post-actions on it // + // unless the defaultGetInterface was used - as it just does a query, and the query will do the post-actions. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(getOutput.getRecord() != null && !usingDefaultGetInterface) { getOutput.setRecord(postRecordActions(getOutput.getRecord())); } @@ -136,6 +130,43 @@ public class GetAction + /******************************************************************************* + ** shorthand way to call for the most common use-case, when you just want the + ** output record to be returned. + *******************************************************************************/ + public QRecord executeForRecord(GetInput getInput) throws QException + { + return (execute(getInput).getRecord()); + } + + + + /******************************************************************************* + ** more shorthand way to call for the most common use-case, when you just want the + ** output record to be returned, and you just want to pass in a table name and primary key. + *******************************************************************************/ + public static QRecord execute(String tableName, Serializable primaryKey) throws QException + { + GetAction getAction = new GetAction(); + GetInput getInput = new GetInput(tableName).withPrimaryKey(primaryKey); + return getAction.executeForRecord(getInput); + } + + + + /******************************************************************************* + ** more shorthand way to call for the most common use-case, when you just want the + ** output record to be returned, and you just want to pass in a table name and unique key + *******************************************************************************/ + public static QRecord execute(String tableName, Map uniqueKey) throws QException + { + GetAction getAction = new GetAction(); + GetInput getInput = new GetInput(tableName).withUniqueKey(uniqueKey); + return getAction.executeForRecord(getInput); + } + + + /******************************************************************************* ** Run a GetAction by using the QueryAction instead (e.g., with a filter made ** from the pkey/ukey, and returning the single record if found). @@ -202,7 +233,7 @@ public class GetAction } else { - throw (new QException("No primaryKey or uniqueKey was passed to Get")); + throw (new QException("Unable to get " + ObjectUtils.tryElse(() -> queryInput.getTable().getLabel(), queryInput.getTableName()) + ". Missing required input.")); } queryInput.setFilter(filter); @@ -216,12 +247,12 @@ public class GetAction ** Run the necessary actions on a record. This may include setting display values, ** translating possible values, and running post-record customizations. *******************************************************************************/ - public QRecord postRecordActions(QRecord record) + public QRecord postRecordActions(QRecord record) throws QException { QRecord returnRecord = record; if(this.postGetRecordCustomizer.isPresent()) { - returnRecord = postGetRecordCustomizer.get().apply(List.of(record)).get(0); + returnRecord = postGetRecordCustomizer.get().postQuery(getInput, List.of(record)).get(0); } if(getInput.getShouldTranslatePossibleValues()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index ea3df9c0..9773d3f0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -37,9 +38,9 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; @@ -61,6 +62,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.DuplicateKeyBadInputStatusMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -168,13 +170,12 @@ public class InsertAction extends AbstractQActionFunction postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole()); + Optional postInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_INSERT_RECORD.getRole()); if(postInsertCustomizer.isPresent()) { try { - postInsertCustomizer.get().setInsertInput(insertInput); - insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords())); + insertOutput.setRecords(postInsertCustomizer.get().postInsert(insertInput, insertOutput.getRecords())); } catch(Exception e) { @@ -232,31 +233,29 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { - preInsertCustomizer.get().setInsertInput(insertInput); - preInsertCustomizer.get().setIsPreview(isPreview); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); } setDefaultValuesInRecords(table, insertInput.getRecords()); ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS); setErrorsIfUniqueKeyErrors(insertInput, table); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS); if(insertInput.getInputSource().shouldValidateRequiredFields()) { validateRequiredFields(insertInput); } - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS); ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); } @@ -290,13 +289,13 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException + private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, boolean isPreview, Optional preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException { if(preInsertCustomizer.isPresent()) { - if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun())) + if(whenToRun.equals(preInsertCustomizer.get().whenToRunPreInsert(insertInput, isPreview))) { - insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords())); + insertInput.setRecords(preInsertCustomizer.get().preInsert(insertInput, insertInput.getRecords(), isPreview)); } } } @@ -321,7 +320,7 @@ public class InsertAction extends AbstractQActionFunction> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get()))) { - record.addError(new BadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table))); + record.addError(new DuplicateKeyBadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table))); foundDupe = true; break; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java index 3834edfb..58cc53a3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java @@ -31,8 +31,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.ActionHelper; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe; @@ -73,7 +73,7 @@ public class QueryAction { private static final QLogger LOG = QLogger.getLogger(QueryAction.class); - private Optional postQueryRecordCustomizer; + private Optional postQueryRecordCustomizer; private QueryInput queryInput; private QueryInterface queryInterface; @@ -100,7 +100,7 @@ public class QueryAction } QBackendMetaData backend = queryInput.getBackend(); - postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole()); + postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole()); this.queryInput = queryInput; if(queryInput.getRecordPipe() != null) @@ -151,6 +151,22 @@ public class QueryAction + /******************************************************************************* + ** shorthand way to call for the most common use-case, when you just want the + ** records to be returned, and you just want to pass in a table name and filter. + *******************************************************************************/ + public static List execute(String tableName, QQueryFilter filter) throws QException + { + QueryAction queryAction = new QueryAction(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(tableName); + queryInput.setFilter(filter); + QueryOutput queryOutput = queryAction.execute(queryInput); + return (queryOutput.getRecords()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -169,6 +185,7 @@ public class QueryAction nextLevelQueryInput.setTableName(association.getAssociatedTableName()); nextLevelQueryInput.setIncludeAssociations(true); nextLevelQueryInput.setAssociationNamesToInclude(buildNextLevelAssociationNamesToInclude(association.getName(), queryInput.getAssociationNamesToInclude())); + nextLevelQueryInput.setTransaction(queryInput.getTransaction()); QQueryFilter filter = new QQueryFilter(); nextLevelQueryInput.setFilter(filter); @@ -264,7 +281,7 @@ public class QueryAction { if(this.postQueryRecordCustomizer.isPresent()) { - records = postQueryRecordCustomizer.get().apply(records); + records = postQueryRecordCustomizer.get().postQuery(queryInput, records); } if(queryInput.getShouldTranslatePossibleValues()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java index 841c6d6b..ca2d1413 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java @@ -47,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -79,9 +80,11 @@ public class ReplaceAction extends AbstractQActionFunction, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey); + Map, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey, allowNullKeyValuesToEqual); + for(QRecord record : page) { - Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual); if(keyValues.isPresent()) { if(existingKeys.containsKey(keyValues.get())) @@ -153,6 +157,17 @@ public class ReplaceAction extends AbstractQActionFunction. + */ + +package com.kingsrook.qqq.backend.core.actions.tables; + + +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; + + +/******************************************************************************* + ** Action to do (generally, "mass") storage operations in a backend. + ** + ** e.g., store a (potentially large) file - specifically - by working with it + ** as either an InputStream or OutputStream. + ** + ** May not be implemented in all backends. + ** + *******************************************************************************/ +public class StorageAction +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + QBackendModuleInterface qBackendModuleInterface = preAction(storageInput); + QStorageInterface storageInterface = qBackendModuleInterface.getStorageInterface(); + return (storageInterface.createOutputStream(storageInput)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public InputStream getInputStream(StorageInput storageInput) throws QException + { + QBackendModuleInterface qBackendModuleInterface = preAction(storageInput); + QStorageInterface storageInterface = qBackendModuleInterface.getStorageInterface(); + return (storageInterface.getInputStream(storageInput)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QBackendModuleInterface preAction(StorageInput storageInput) throws QException + { + ActionHelper.validateSession(storageInput); + + if(storageInput.getTableName() == null) + { + throw (new QException("Table name was not specified in storage input")); + } + + QTableMetaData table = storageInput.getTable(); + if(table == null) + { + throw (new QException("A table named [" + storageInput.getTableName() + "] was not found in the active QInstance")); + } + + QBackendMetaData backend = storageInput.getBackend(); + QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); + QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend); + return (qModule); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void makePublic(StorageInput storageInput) throws QException + { + QBackendModuleInterface qBackendModuleInterface = preAction(storageInput); + QStorageInterface storageInterface = qBackendModuleInterface.getStorageInterface(); + storageInterface.makePublic(storageInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public String getDownloadURL(StorageInput storageInput) throws QException + { + QBackendModuleInterface qBackendModuleInterface = preAction(storageInput); + QStorageInterface storageInterface = qBackendModuleInterface.getStorageInterface(); + return (storageInterface.getDownloadURL(storageInput)); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java index a69acc45..2b64c1d9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.tables; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,9 +35,8 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper; @@ -69,6 +69,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -192,14 +193,12 @@ public class UpdateAction ////////////////////////////////////////////////////////////// // finally, run the post-update customizer, if there is one // ////////////////////////////////////////////////////////////// - Optional postUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPostUpdateCustomizer.class, table, TableCustomizers.POST_UPDATE_RECORD.getRole()); + Optional postUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_UPDATE_RECORD.getRole()); if(postUpdateCustomizer.isPresent()) { try { - postUpdateCustomizer.get().setUpdateInput(updateInput); - oldRecordList.ifPresent(l -> postUpdateCustomizer.get().setOldRecordList(l)); - updateOutput.setRecords(postUpdateCustomizer.get().apply(updateOutput.getRecords())); + updateOutput.setRecords(postUpdateCustomizer.get().postUpdate(updateInput, updateOutput.getRecords(), oldRecordList)); } catch(Exception e) { @@ -273,13 +272,10 @@ public class UpdateAction /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-update customizer, if there is one // /////////////////////////////////////////////////////////////////////////// - Optional preUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPreUpdateCustomizer.class, table, TableCustomizers.PRE_UPDATE_RECORD.getRole()); + Optional preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole()); if(preUpdateCustomizer.isPresent()) { - preUpdateCustomizer.get().setUpdateInput(updateInput); - preUpdateCustomizer.get().setIsPreview(isPreview); - oldRecordList.ifPresent(l -> preUpdateCustomizer.get().setOldRecordList(l)); - updateInput.setRecords(preUpdateCustomizer.get().apply(updateInput.getRecords())); + updateInput.setRecords(preUpdateCustomizer.get().preUpdate(updateInput, updateInput.getRecords(), isPreview, oldRecordList)); } } @@ -340,6 +336,9 @@ public class UpdateAction QTableMetaData table = updateInput.getTable(); QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField()); + ///////////////////////////////////////////////////////////// + // todo - evolve to use lock tree (e.g., from multi-locks) // + ///////////////////////////////////////////////////////////// List onlyWriteLocks = RecordSecurityLockFilters.filterForOnlyWriteLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())); for(List page : CollectionUtils.getPages(updateInput.getRecords(), 1000)) @@ -399,7 +398,12 @@ public class UpdateAction QRecord oldRecord = lookedUpRecords.get(value); QFieldType fieldType = table.getField(lock.getFieldName()).getType(); Serializable lockValue = ValueUtils.getValueAsFieldType(fieldType, oldRecord.getValue(lock.getFieldName())); - ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, record, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE); + + List errors = ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE, Collections.emptyMap()); + if(CollectionUtils.nullSafeHasContents(errors)) + { + errors.forEach(e -> record.addError(e)); + } } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java index e32caf6e..7cbc2fa6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java @@ -214,71 +214,78 @@ public class QueryStatManager *******************************************************************************/ public void add(QueryStat queryStat) { - if(queryStat == null) + try { - return; - } - - if(active) - { - //////////////////////////////////////////////////////////////////////////////////////// - // set fields that we need to capture now (rather than when the thread to store runs) // - //////////////////////////////////////////////////////////////////////////////////////// - if(queryStat.getFirstResultTimestamp() == null) + if(queryStat == null) { - queryStat.setFirstResultTimestamp(Instant.now()); - } - - if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null) - { - long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli(); - queryStat.setFirstResultMillis((int) millis); - } - - if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore) - { - ////////////////////////////////////////////////////////////// - // discard this record if it's under the min millis setting // - ////////////////////////////////////////////////////////////// return; } - if(queryStat.getSessionId() == null && QContext.getQSession() != null) + if(active) { - queryStat.setSessionId(QContext.getQSession().getUuid()); - } - - if(queryStat.getAction() == null) - { - if(!QContext.getActionStack().isEmpty()) + //////////////////////////////////////////////////////////////////////////////////////// + // set fields that we need to capture now (rather than when the thread to store runs) // + //////////////////////////////////////////////////////////////////////////////////////// + if(queryStat.getFirstResultTimestamp() == null) { - queryStat.setAction(QContext.getActionStack().peek().getActionIdentity()); + queryStat.setFirstResultTimestamp(Instant.now()); } - else + + if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null) { - boolean expected = false; - Exception e = new Exception("Unexpected empty action stack"); - for(StackTraceElement stackTraceElement : e.getStackTrace()) + long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli(); + queryStat.setFirstResultMillis((int) millis); + } + + if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore) + { + ////////////////////////////////////////////////////////////// + // discard this record if it's under the min millis setting // + ////////////////////////////////////////////////////////////// + return; + } + + if(queryStat.getSessionId() == null && QContext.getQSession() != null) + { + queryStat.setSessionId(QContext.getQSession().getUuid()); + } + + if(queryStat.getAction() == null) + { + if(QContext.getActionStack() != null && !QContext.getActionStack().isEmpty()) { - String className = stackTraceElement.getClassName(); - if(className.contains(QueryStatManagerInsertJob.class.getName())) + queryStat.setAction(QContext.getActionStack().peek().getActionIdentity()); + } + else + { + boolean expected = false; + Exception e = new Exception("Unexpected empty action stack"); + for(StackTraceElement stackTraceElement : e.getStackTrace()) { - expected = true; - break; + String className = stackTraceElement.getClassName(); + if(className.contains(QueryStatManagerInsertJob.class.getName())) + { + expected = true; + break; + } + } + + if(!expected) + { + LOG.debug(e); } } + } - if(!expected) - { - LOG.debug(e); - } + synchronized(this) + { + queryStats.add(queryStat); } } - - synchronized(this) - { - queryStats.add(queryStat); - } + } + catch(Exception e) + { + LOG.debug("Error adding query stat", e); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java index adab071e..7832344e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java @@ -54,7 +54,7 @@ public class UniqueKeyHelper /******************************************************************************* ** *******************************************************************************/ - public static Map, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List recordList, UniqueKey uniqueKey) throws QException + public static Map, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List recordList, UniqueKey uniqueKey, boolean allowNullKeyValuesToEqual) throws QException { List ukFieldNames = uniqueKey.getFieldNames(); Map, Serializable> existingRecords = new HashMap<>(); @@ -112,7 +112,7 @@ public class UniqueKeyHelper QueryOutput queryOutput = new QueryAction().execute(queryInput); for(QRecord record : queryOutput.getRecords()) { - Optional> keyValues = getKeyValues(table, uniqueKey, record); + Optional> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual); if(keyValues.isPresent()) { existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField())); @@ -128,7 +128,17 @@ public class UniqueKeyHelper /******************************************************************************* ** *******************************************************************************/ - public static Optional> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record) + public static Map, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List recordList, UniqueKey uniqueKey) throws QException + { + return (getExistingKeys(transaction, table, recordList, uniqueKey, false)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Optional> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record, boolean allowNullKeyValuesToEqual) { try { @@ -138,7 +148,19 @@ public class UniqueKeyHelper QFieldMetaData field = table.getField(fieldName); Serializable value = record.getValue(fieldName); Serializable typedValue = ValueUtils.getValueAsFieldType(field.getType(), value); - keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue); + + /////////////////////////////////////////////////////////////////////////////////// + // if null value, look at flag to determine if a null should be used (which will // + // allow keys to match), or a NullUniqueKeyValue, (which will never match) // + /////////////////////////////////////////////////////////////////////////////////// + if(typedValue == null) + { + keyValues.add(allowNullKeyValuesToEqual ? null : new NullUniqueKeyValue()); + } + else + { + keyValues.add(typedValue); + } } return (Optional.of(keyValues)); } @@ -150,6 +172,16 @@ public class UniqueKeyHelper + /******************************************************************************* + ** + *******************************************************************************/ + public static Optional> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record) + { + return (getKeyValues(table, uniqueKey, record, false)); + } + + + /******************************************************************************* ** To make a list of unique key values here behave like they do in an RDBMS ** (which is what we're trying to mimic - which is - 2 null values in a field diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java index 1f235999..dcc730d9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.tables.helpers; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,13 +43,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.NullValueBehaviorUtil; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -81,160 +85,273 @@ public class ValidateRecordSecurityLockHelper *******************************************************************************/ public static void validateSecurityFields(QTableMetaData table, List records, Action action) throws QException { - List locksToCheck = getRecordSecurityLocks(table, action); - if(CollectionUtils.nullSafeIsEmpty(locksToCheck)) + MultiRecordSecurityLock locksToCheck = getRecordSecurityLocks(table, action); + if(locksToCheck == null || CollectionUtils.nullSafeIsEmpty(locksToCheck.getLocks())) + { + return; + } + + ////////////////////////////////////////////////////////////////////////////////////////// + // we will be relying on primary keys being set in records - but (at least for inserts) // + // we might not have pkeys - so make them up (and clear them out at the end) // + ////////////////////////////////////////////////////////////////////////////////////////// + Map madeUpPrimaryKeys = makeUpPrimaryKeysIfNeeded(records, table); + + //////////////////////////////// + // actually check lock values // + //////////////////////////////// + Map errorRecords = new HashMap<>(); + evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys); + + ///////////////////////////////// + // propagate errors to records // + ///////////////////////////////// + for(RecordWithErrors recordWithErrors : errorRecords.values()) + { + recordWithErrors.propagateErrorsToRecord(locksToCheck); + } + + ///////////////////////////////// + // remove made-up primary keys // + ///////////////////////////////// + String primaryKeyField = table.getPrimaryKeyField(); + for(QRecord record : madeUpPrimaryKeys.values()) + { + record.setValue(primaryKeyField, null); + } + } + + + + /******************************************************************************* + ** For a list of `records` from a `table`, and a given `action`, evaluate a + ** `recordSecurityLock` (which may be a multi-lock) - populating the input map + ** of `errorRecords` - key'ed by primary key value (real or made up), with + ** error messages existing in a tree, with positions matching the multi-lock + ** tree that we're navigating, as tracked by `treePosition`. + ** + ** Recursively processes multi-locks (and the top-level call is always with a + ** multi-lock - as the table's recordLocks list converted to an AND-multi-lock). + ** + ** Of note - for the case of READ_WRITE locks, we're only evaluating the values + ** on the record, to see if they're allowed for us to store (because if we didn't + ** have the key, we wouldn't have been able to read the value (which is verified + ** outside of here, in UpdateAction/DeleteAction). + ** + ** BUT - WRITE locks - in their case, we read the record no matter what, and in + ** here we need to verify we have a key that allows us to WRITE the record. + *******************************************************************************/ + private static void evaluateRecordLocks(QTableMetaData table, List records, Action action, RecordSecurityLock recordSecurityLock, Map errorRecords, List treePosition, Map madeUpPrimaryKeys) throws QException + { + if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock) + { + ///////////////////////////////////////////// + // for multi-locks, make recursive descent // + ///////////////////////////////////////////// + int i = 0; + for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks())) + { + treePosition.add(i); + evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys); + treePosition.remove(treePosition.size() - 1); + i++; + } + + return; + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this lock has an all-access key, and the user has that key, then there can't be any errors here, so return early // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); + if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN)) { return; } //////////////////////////////// - // actually check lock values // + // proceed w/ non-multi locks // //////////////////////////////// - for(RecordSecurityLock recordSecurityLock : locksToCheck) + String primaryKeyField = table.getPrimaryKeyField(); + if(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain())) { - if(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain())) + ////////////////////////////////////////////////////////////////////////////////// + // handle the value being in the table we're inserting/updating (e.g., no join) // + ////////////////////////////////////////////////////////////////////////////////// + QFieldMetaData field = table.getField(recordSecurityLock.getFieldName()); + + for(QRecord record : records) { - ////////////////////////////////////////////////////////////////////////////////// - // handle the value being in the table we're inserting/updating (e.g., no join) // - ////////////////////////////////////////////////////////////////////////////////// - QFieldMetaData field = table.getField(recordSecurityLock.getFieldName()); - - for(QRecord record : records) + if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope())) { - if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope())) - { - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - // if this is a read-write lock, then if we have the record, it means we were able to read the record. // - // So if we're not updating the security field, then no error can come from it! // - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - continue; - } + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this is a read-write lock, then if we have the record, it means we were able to read the record. // + // So if we're not updating the security field, then no error can come from it! // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + continue; + } - Serializable recordSecurityValue = record.getValue(field.getName()); - validateRecordSecurityValue(table, record, recordSecurityLock, recordSecurityValue, field.getType(), action); + Serializable recordSecurityValue = record.getValue(field.getName()); + List recordErrors = validateRecordSecurityValue(table, recordSecurityLock, recordSecurityValue, field.getType(), action, madeUpPrimaryKeys); + if(CollectionUtils.nullSafeHasContents(recordErrors)) + { + errorRecords.computeIfAbsent(record.getValue(primaryKeyField), (k) -> new RecordWithErrors(record)).addAll(recordErrors, treePosition); } } - else + } + else + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else look for the joined record - if it isn't found, assume a fail - else validate security value if found // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QJoinMetaData leftMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0)); + QJoinMetaData rightMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1)); + + //////////////////////////////// + // todo probably, but more... // + //////////////////////////////// + // if(leftMostJoin.getLeftTable().equals(table.getName())) + // { + // leftMostJoin = leftMostJoin.flip(); + // } + + QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable()); + QTableMetaData leftMostJoinTable = QContext.getQInstance().getTable(leftMostJoin.getLeftTable()); + + for(List inputRecordPage : CollectionUtils.getPages(records, 500)) { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // else look for the joined record - if it isn't found, assume a fail - else validate security value if found // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QJoinMetaData leftMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0)); - QJoinMetaData rightMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1)); - QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable()); - QTableMetaData leftMostJoinTable = QContext.getQInstance().getTable(leftMostJoin.getLeftTable()); + //////////////////////////////////////////////////////////////////////////////////////////////// + // set up a query for joined records // + // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) // + //////////////////////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(leftMostJoin.getLeftTable()); + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); + queryInput.setFilter(filter); - for(List inputRecordPage : CollectionUtils.getPages(records, 500)) + for(String joinName : recordSecurityLock.getJoinNameChain()) { - //////////////////////////////////////////////////////////////////////////////////////////////// - // set up a query for joined records // - // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) // - //////////////////////////////////////////////////////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(leftMostJoin.getLeftTable()); - QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); - queryInput.setFilter(filter); - - for(String joinName : recordSecurityLock.getJoinNameChain()) + /////////////////////////////////////// + // we don't need the right-most join // + /////////////////////////////////////// + if(!joinName.equals(rightMostJoin.getName())) { - /////////////////////////////////////// - // we don't need the right-most join // - /////////////////////////////////////// - if(!joinName.equals(rightMostJoin.getName())) + queryInput.withQueryJoin(new QueryJoin().withJoinMetaData(QContext.getQInstance().getJoin(joinName)).withSelect(true)); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // foreach input record (in this page), put it in a listing hash, with key = list of join-values // + // e.g., (17,47)=(QRecord1), (18,48)=(QRecord2,QRecord3) // + // also build up the query's sub-filters here (only adding them if they're unique). // + // e.g., 2 order-lines referencing the same orderId don't need to be added to the query twice // + /////////////////////////////////////////////////////////////////////////////////////////////////// + ListingHash, QRecord> inputRecordMapByJoinFields = new ListingHash<>(); + for(QRecord inputRecord : inputRecordPage) + { + List inputRecordJoinValues = new ArrayList<>(); + QQueryFilter subFilter = new QQueryFilter(); + + boolean updatingAnyLockJoinFields = false; + for(JoinOn joinOn : rightMostJoin.getJoinOns()) + { + QFieldType type = rightMostJoinTable.getField(joinOn.getRightField()).getType(); + Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField())); + inputRecordJoinValues.add(inputRecordValue); + + // if we have a value in this field (and it's not the primary key), then it means we're updating part of the lock + if(inputRecordValue != null && !joinOn.getRightField().equals(table.getPrimaryKeyField())) { - queryInput.withQueryJoin(new QueryJoin().withJoinMetaData(QContext.getQInstance().getJoin(joinName)).withSelect(true)); + updatingAnyLockJoinFields = true; } + + subFilter.addCriteria(inputRecordValue == null + ? new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.IS_BLANK) + : new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.EQUALS, inputRecordValue)); } - /////////////////////////////////////////////////////////////////////////////////////////////////// - // foreach input record (in this page), put it in a listing hash, with key = list of join-values // - // e.g., (17,47)=(QRecord1), (18,48)=(QRecord2,QRecord3) // - // also build up the query's sub-filters here (only adding them if they're unique). // - // e.g., 2 order-lines referencing the same orderId don't need to be added to the query twice // - /////////////////////////////////////////////////////////////////////////////////////////////////// - ListingHash, QRecord> inputRecordMapByJoinFields = new ListingHash<>(); - for(QRecord inputRecord : inputRecordPage) + ////////////////////////////////// + // todo maybe, some version of? // + ////////////////////////////////// + // if(action.equals(Action.UPDATE) && !updatingAnyLockJoinFields && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope())) + // { + // ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // // if this is a read-write lock, then if we have the record, it means we were able to read the record. // + // // So if we're not updating the security field, then no error can come from it! // + // ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // continue; + // } + + if(!inputRecordMapByJoinFields.containsKey(inputRecordJoinValues)) { - List inputRecordJoinValues = new ArrayList<>(); - QQueryFilter subFilter = new QQueryFilter(); - - for(JoinOn joinOn : rightMostJoin.getJoinOns()) - { - QFieldType type = rightMostJoinTable.getField(joinOn.getRightField()).getType(); - Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField())); - inputRecordJoinValues.add(inputRecordValue); - - subFilter.addCriteria(inputRecordValue == null - ? new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.IS_BLANK) - : new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.EQUALS, inputRecordValue)); - } - - if(!inputRecordMapByJoinFields.containsKey(inputRecordJoinValues)) - { - //////////////////////////////////////////////////////////////////////////////// - // only add this sub-filter if it's for a list of keys we haven't seen before // - //////////////////////////////////////////////////////////////////////////////// - filter.addSubFilter(subFilter); - } - - inputRecordMapByJoinFields.add(inputRecordJoinValues, inputRecord); + //////////////////////////////////////////////////////////////////////////////// + // only add this sub-filter if it's for a list of keys we haven't seen before // + //////////////////////////////////////////////////////////////////////////////// + filter.addSubFilter(subFilter); } - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // execute the query for joined records - then put them in a map with keys corresponding to the join values // - // e.g., (17,47)=(JoinRecord), (18,48)=(JoinRecord) // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QueryOutput queryOutput = new QueryAction().execute(queryInput); - Map, QRecord> joinRecordMapByJoinFields = new HashMap<>(); - for(QRecord joinRecord : queryOutput.getRecords()) - { - List joinRecordValues = new ArrayList<>(); - for(JoinOn joinOn : rightMostJoin.getJoinOns()) - { - Serializable joinValue = joinRecord.getValue(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField()); - if(joinValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> !n.contains("."))) - { - joinValue = joinRecord.getValue(joinOn.getLeftField()); - } - joinRecordValues.add(joinValue); - } + inputRecordMapByJoinFields.add(inputRecordJoinValues, inputRecord); + } - joinRecordMapByJoinFields.put(joinRecordValues, joinRecord); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // execute the query for joined records - then put them in a map with keys corresponding to the join values // + // e.g., (17,47)=(JoinRecord), (18,48)=(JoinRecord) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Map, QRecord> joinRecordMapByJoinFields = new HashMap<>(); + for(QRecord joinRecord : queryOutput.getRecords()) + { + List joinRecordValues = new ArrayList<>(); + for(JoinOn joinOn : rightMostJoin.getJoinOns()) + { + Serializable joinValue = joinRecord.getValue(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField()); + if(joinValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> !n.contains("."))) + { + joinValue = joinRecord.getValue(joinOn.getLeftField()); + } + joinRecordValues.add(joinValue); } - ////////////////////////////////////////////////////////////////////////////////////////////////// - // now for each input record, look for its joinRecord - if it isn't found, then this insert // - // isn't allowed. if it is found, then validate its value matches this session's security keys // - ////////////////////////////////////////////////////////////////////////////////////////////////// - for(Map.Entry, List> entry : inputRecordMapByJoinFields.entrySet()) + joinRecordMapByJoinFields.put(joinRecordValues, joinRecord); + } + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // now for each input record, look for its joinRecord - if it isn't found, then this insert // + // isn't allowed. if it is found, then validate its value matches this session's security keys // + ////////////////////////////////////////////////////////////////////////////////////////////////// + for(Map.Entry, List> entry : inputRecordMapByJoinFields.entrySet()) + { + List inputRecordJoinValues = entry.getKey(); + List inputRecords = entry.getValue(); + if(joinRecordMapByJoinFields.containsKey(inputRecordJoinValues)) { - List inputRecordJoinValues = entry.getKey(); - List inputRecords = entry.getValue(); - if(joinRecordMapByJoinFields.containsKey(inputRecordJoinValues)) + QRecord joinRecord = joinRecordMapByJoinFields.get(inputRecordJoinValues); + + String fieldName = recordSecurityLock.getFieldName().replaceFirst(".*\\.", ""); + QFieldMetaData field = leftMostJoinTable.getField(fieldName); + Serializable recordSecurityValue = joinRecord.getValue(fieldName); + if(recordSecurityValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> n.contains("."))) { - QRecord joinRecord = joinRecordMapByJoinFields.get(inputRecordJoinValues); + recordSecurityValue = joinRecord.getValue(recordSecurityLock.getFieldName()); + } - String fieldName = recordSecurityLock.getFieldName().replaceFirst(".*\\.", ""); - QFieldMetaData field = leftMostJoinTable.getField(fieldName); - Serializable recordSecurityValue = joinRecord.getValue(fieldName); - if(recordSecurityValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> n.contains("."))) + for(QRecord inputRecord : inputRecords) + { + List recordErrors = validateRecordSecurityValue(table, recordSecurityLock, recordSecurityValue, field.getType(), action, madeUpPrimaryKeys); + if(CollectionUtils.nullSafeHasContents(recordErrors)) { - recordSecurityValue = joinRecord.getValue(recordSecurityLock.getFieldName()); - } - - for(QRecord inputRecord : inputRecords) - { - validateRecordSecurityValue(table, inputRecord, recordSecurityLock, recordSecurityValue, field.getType(), action); + errorRecords.computeIfAbsent(inputRecord.getValue(primaryKeyField), (k) -> new RecordWithErrors(inputRecord)).addAll(recordErrors, treePosition); } } - else + } + else + { + for(QRecord inputRecord : inputRecords) { - for(QRecord inputRecord : inputRecords) + if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock))) { - if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock))) - { - inputRecord.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found.")); - } + PermissionDeniedMessage error = new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found."); + errorRecords.computeIfAbsent(inputRecord.getValue(primaryKeyField), (k) -> new RecordWithErrors(inputRecord)).add(error, treePosition); } } } @@ -246,45 +363,80 @@ public class ValidateRecordSecurityLockHelper /******************************************************************************* - ** + ** for tracking errors, we use primary keys. add "made up" ones to records + ** if needed (e.g., insert use-case). *******************************************************************************/ - private static List getRecordSecurityLocks(QTableMetaData table, Action action) + private static Map makeUpPrimaryKeysIfNeeded(List records, QTableMetaData table) { - List recordSecurityLocks = CollectionUtils.nonNullList(table.getRecordSecurityLocks()); - List locksToCheck = new ArrayList<>(); - - recordSecurityLocks = switch(action) + String primaryKeyField = table.getPrimaryKeyField(); + Map madeUpPrimaryKeys = new HashMap<>(); + Integer madeUpPrimaryKey = Integer.MIN_VALUE / 2; + for(QRecord record : records) { - case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLocks(recordSecurityLocks); - case SELECT -> RecordSecurityLockFilters.filterForReadLocks(recordSecurityLocks); + if(record.getValue(primaryKeyField) == null) + { + madeUpPrimaryKeys.put(madeUpPrimaryKey, record); + record.setValue(primaryKeyField, madeUpPrimaryKey); + madeUpPrimaryKey++; + } + } + return madeUpPrimaryKeys; + } + + + + /******************************************************************************* + ** For a given table & action type, convert the table's record locks to a + ** MultiRecordSecurityLock, with only the appropriate lock-scopes being included + ** (e.g., read-locks for selects, write-locks for insert/update/delete). + *******************************************************************************/ + static MultiRecordSecurityLock getRecordSecurityLocks(QTableMetaData table, Action action) + { + List allLocksOnTable = CollectionUtils.nonNullList(table.getRecordSecurityLocks()); + MultiRecordSecurityLock locksOfType = switch(action) + { + case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLockTree(allLocksOnTable); + case SELECT -> RecordSecurityLockFilters.filterForReadLockTree(allLocksOnTable); default -> throw (new IllegalArgumentException("Unsupported action: " + action)); }; + if(action.equals(Action.UPDATE)) + { + //////////////////////////////////////////////////////////////////////////// + // todo at some point this seemed right, but now it doesn't - needs work. // + //////////////////////////////////////////////////////////////////////////// + // //////////////////////////////////////////////////////// + // // when doing an update, convert all OR's to AND's... // + // //////////////////////////////////////////////////////// + // updateOperators(locksOfType, MultiRecordSecurityLock.BooleanOperator.AND); + } + //////////////////////////////////////// // if there are no locks, just return // //////////////////////////////////////// - if(CollectionUtils.nullSafeIsEmpty(recordSecurityLocks)) + if(locksOfType == null || CollectionUtils.nullSafeIsEmpty(locksOfType.getLocks())) { return (null); } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // decide if any locks need checked - where one may not need checked if it has an all-access key, and the user has all-access // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - for(RecordSecurityLock recordSecurityLock : recordSecurityLocks) + return (locksOfType); + } + + + + /******************************************************************************* + ** for a full multi-lock tree, set all of the boolean operators to the specified one. + *******************************************************************************/ + private static void updateOperators(MultiRecordSecurityLock multiLock, MultiRecordSecurityLock.BooleanOperator operator) + { + multiLock.setOperator(operator); + for(RecordSecurityLock childLock : multiLock.getLocks()) { - QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); - if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN)) + if(childLock instanceof MultiRecordSecurityLock childMultiLock) { - LOG.trace("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock."); - } - else - { - locksToCheck.add(recordSecurityLock); + updateOperators(childMultiLock, operator); } } - - return (locksToCheck); } @@ -292,9 +444,9 @@ public class ValidateRecordSecurityLockHelper /******************************************************************************* ** *******************************************************************************/ - public static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action) + public static List validateRecordSecurityValue(QTableMetaData table, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action, Map madeUpPrimaryKeys) { - if(recordSecurityValue == null) + if(recordSecurityValue == null || (madeUpPrimaryKeys != null && madeUpPrimaryKeys.containsKey(recordSecurityValue))) { ///////////////////////////////////////////////////////////////// // handle null values - error if the NullValueBehavior is DENY // @@ -302,7 +454,7 @@ public class ValidateRecordSecurityLockHelper if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock))) { String lockLabel = CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()) ? recordSecurityLock.getSecurityKeyType() : table.getField(recordSecurityLock.getFieldName()).getLabel(); - record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel)); + return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel))); } } else @@ -314,15 +466,305 @@ public class ValidateRecordSecurityLockHelper /////////////////////////////////////////////////////////////////////////////////////////////// // avoid telling the user a value from a foreign record that they didn't pass in themselves. // /////////////////////////////////////////////////////////////////////////////////////////////// - record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record.")); + return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record."))); } else { QFieldMetaData field = table.getField(recordSecurityLock.getFieldName()); - record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel())); + return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel()))); } } } + return (Collections.emptyList()); + } + + + + /******************************************************************************* + ** Class to track errors that we're associating with a record. + ** + ** More complex than it first seems to be needed, because as we're evaluating + ** locks, we might find some, but based on the boolean condition associated with + ** them, they might not actually be record-level errors. + ** + ** e.g., two locks with an OR relationship - as long as one passes, the record + ** should have no errors. And so-on through the tree of locks/multi-locks. + ** + ** Stores the errors in a tree of ErrorTreeNode objects. + ** + ** References into that tree are achieved via a List of Integer called "tree positions" + ** where each entry in the list denotes the index of the tree node at that level. + ** + ** e.g., given this tree: + ** 
+    **   A      B
+    **  / \    /|\
+    ** C   D  E F G
+    **     |
+    **     H
+    **
+ ** + ** The positions of each node would be: + ** 
+    ** A: [0]
+    ** B: [1]
+    ** C: [0,0]
+    ** D: [0,1]
+    ** E: [1,0]
+    ** F: [1,1]
+    ** G: [1,2]
+    ** H: [0,1,0]
+    **
+ *******************************************************************************/ + static class RecordWithErrors + { + private QRecord record; + private ErrorTreeNode errorTree; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RecordWithErrors(QRecord record) + { + this.record = record; + } + + + + /******************************************************************************* + ** add a list of errors, for a given list of tree positions + *******************************************************************************/ + public void addAll(List recordErrors, List treePositions) + { + if(errorTree == null) + { + errorTree = new ErrorTreeNode(); + } + + ErrorTreeNode node = errorTree; + for(Integer treePosition : treePositions) + { + if(node.children == null) + { + node.children = new ArrayList<>(treePosition); + } + + while(treePosition >= node.children.size()) + { + node.children.add(null); + } + + if(node.children.get(treePosition) == null) + { + node.children.set(treePosition, new ErrorTreeNode()); + } + + node = node.children.get(treePosition); + } + + if(node.errors == null) + { + node.errors = new ArrayList<>(); + } + node.errors.addAll(recordErrors); + } + + + + /******************************************************************************* + ** add a single error to a given tree-position + *******************************************************************************/ + public void add(QErrorMessage error, List treePositions) + { + addAll(List.of(error), treePositions); + } + + + + /******************************************************************************* + ** after the tree of errors has been built - walk a lock-tree (locksToCheck) + ** and resolve boolean operations, to get a final list of errors (possibly empty) + ** to put on the record. + *******************************************************************************/ + public void propagateErrorsToRecord(MultiRecordSecurityLock locksToCheck) + { + List errors = recursivePropagation(locksToCheck, new ArrayList<>()); + + if(CollectionUtils.nullSafeHasContents(errors)) + { + errors.forEach(e -> record.addError(e)); + } + } + + + + /******************************************************************************* + ** recursive implementation of the propagation method - e.g., walk tree applying + ** boolean logic. + *******************************************************************************/ + private List recursivePropagation(MultiRecordSecurityLock locksToCheck, List treePositions) + { + ////////////////////////////////////////////////////////////////// + // build a list of errors at this level (and deeper levels too) // + ////////////////////////////////////////////////////////////////// + List errorsFromThisLevel = new ArrayList<>(); + + int i = 0; + for(RecordSecurityLock lock : locksToCheck.getLocks()) + { + List errorsFromThisLock; + + treePositions.add(i); + if(lock instanceof MultiRecordSecurityLock childMultiLock) + { + errorsFromThisLock = recursivePropagation(childMultiLock, treePositions); + } + else + { + errorsFromThisLock = getErrorsFromTree(treePositions); + } + + errorsFromThisLevel.addAll(errorsFromThisLock); + + treePositions.remove(treePositions.size() - 1); + i++; + } + + if(MultiRecordSecurityLock.BooleanOperator.AND.equals(locksToCheck.getOperator())) + { + ////////////////////////////////////////////////////////////// + // for an AND - if there were ANY errors, then return them. // + ////////////////////////////////////////////////////////////// + if(!errorsFromThisLevel.isEmpty()) + { + return (errorsFromThisLevel); + } + } + else // OR + { + ////////////////////////////////////////////////////////// + // for an OR - only return if ALL conditions had errors // + ////////////////////////////////////////////////////////// + if(errorsFromThisLevel.size() == locksToCheck.getLocks().size()) + { + return (errorsFromThisLevel); // todo something smarter? + } + } + + /////////////////////////////////// + // else - no errors - empty list // + /////////////////////////////////// + return Collections.emptyList(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List getErrorsFromTree(List treePositions) + { + ErrorTreeNode node = errorTree; + + for(Integer treePosition : treePositions) + { + if(node.children == null) + { + return Collections.emptyList(); + } + + if(treePosition >= node.children.size()) + { + return Collections.emptyList(); + } + + if(node.children.get(treePosition) == null) + { + return Collections.emptyList(); + } + + node = node.children.get(treePosition); + } + + if(node.errors == null) + { + return Collections.emptyList(); + } + + return node.errors; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + try + { + return JsonUtils.toPrettyJson(this); + } + catch(Exception e) + { + return "error in toString"; + } + } + } + + + + /******************************************************************************* + ** tree node used by RecordWithErrors + *******************************************************************************/ + static class ErrorTreeNode + { + private List errors; + private ArrayList children; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + try + { + return JsonUtils.toPrettyJson(this); + } + catch(Exception e) + { + return "error in toString"; + } + } + + + + /******************************************************************************* + ** Getter for errors - only here for Jackson/toString + ** + *******************************************************************************/ + public List getErrors() + { + return errors; + } + + + + /******************************************************************************* + ** Getter for children - only here for Jackson/toString + ** + *******************************************************************************/ + public ArrayList getChildren() + { + return children; + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 12a3f2f9..8fac20fc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -118,6 +118,7 @@ public class QPossibleValueTranslator } + /******************************************************************************* ** Constructor ** @@ -269,6 +270,10 @@ public class QPossibleValueTranslator { value = ValueUtils.getValueAsInteger(value); } + if(field.getType().equals(QFieldType.LONG) && !(value instanceof Long)) + { + value = ValueUtils.getValueAsLong(value); + } } catch(QValueException e) { @@ -366,6 +371,14 @@ public class QPossibleValueTranslator *******************************************************************************/ private String translatePossibleValueCustom(Serializable value, QPossibleValueSource possibleValueSource) { + ///////////////////////////////// + // null input gets null output // + ///////////////////////////////// + if(value == null) + { + return (null); + } + try { QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource); @@ -409,7 +422,6 @@ public class QPossibleValueTranslator /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:Indentation") private String doFormatPossibleValue(String formatString, List valueFields, Object id, String label) { List values = new ArrayList<>(); @@ -548,20 +560,47 @@ public class QPossibleValueTranslator *******************************************************************************/ private void primePvsCache(String tableName, List possibleValueSources, Collection values) { + String idField = null; for(QPossibleValueSource possibleValueSource : possibleValueSources) { possibleValueCache.putIfAbsent(possibleValueSource.getName(), new HashMap<>()); + String thisPvsIdField; + if(StringUtils.hasContent(possibleValueSource.getOverrideIdField())) + { + thisPvsIdField = possibleValueSource.getOverrideIdField(); + } + else + { + thisPvsIdField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField(); + } + + if(idField == null) + { + idField = thisPvsIdField; + } + else + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // does this ever happen? maybe not... because, like, the list of values probably wouldn't make sense for // + // more than one field in the table... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!idField.equals(thisPvsIdField)) + { + for(QPossibleValueSource valueSource : possibleValueSources) + { + primePvsCache(tableName, List.of(valueSource), values); + } + } + } } try { - String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField(); - for(List page : CollectionUtils.getPages(values, 1000)) { QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page))); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(idField, QCriteriaOperator.IN, page))); queryInput.setTransaction(getTransaction(tableName)); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -606,7 +645,7 @@ public class QPossibleValueTranslator /////////////////////////////////////////////////////////////////////////////////// for(QRecord record : queryOutput.getRecords()) { - Serializable pkeyValue = record.getValue(primaryKeyField); + Serializable pkeyValue = record.getValue(idField); for(QPossibleValueSource possibleValueSource : possibleValueSources) { QPossibleValue possibleValue = new QPossibleValue<>(pkeyValue, record.getRecordLabel()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 5b176bbc..fab61595 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -28,7 +28,6 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -364,7 +363,9 @@ public class QValueFormatter } } - setDisplayValuesInRecord(fieldMap, record); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null); + + setDisplayValuesInRecord(table, fieldMap, record, true); record.setRecordLabel(formatRecordLabel(table, record)); } } @@ -374,61 +375,49 @@ public class QValueFormatter /******************************************************************************* ** For a list of records, set their recordLabels and display values *******************************************************************************/ - public static void setDisplayValuesInRecords(Collection fields, List records) + public static void setDisplayValuesInRecords(QTableMetaData table, Map fields, List records) { if(records == null) { return; } + if(table != null) + { + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null); + } + for(QRecord record : records) { - setDisplayValuesInRecord(fields, record); + setDisplayValuesInRecord(table, fields, record, true); } } /******************************************************************************* - ** For a list of records, set their recordLabels and display values + ** For a single record, set its display values - public version of this. *******************************************************************************/ - public static void setDisplayValuesInRecords(Map fields, List records) + public static void setDisplayValuesInRecord(QTableMetaData table, Map fields, QRecord record) { - if(records == null) - { - return; - } - - for(QRecord record : records) - { - setDisplayValuesInRecord(fields, record); - } + setDisplayValuesInRecord(table, fields, record, false); } - /******************************************************************************* - ** For a list of records, set their display values + ** For a single record, set its display values - where caller (meant to stay private) + ** can specify if they've already done fieldBehaviors (to avoid re-doing). *******************************************************************************/ - public static void setDisplayValuesInRecord(Collection fields, QRecord record) + private static void setDisplayValuesInRecord(QTableMetaData table, Map fields, QRecord record, boolean alreadyAppliedFieldDisplayBehaviors) { - for(QFieldMetaData field : fields) + if(!alreadyAppliedFieldDisplayBehaviors) { - if(record.getDisplayValue(field.getName()) == null) + if(table != null) { - String formattedValue = formatValue(field, record.getValue(field.getName())); - record.setDisplayValue(field.getName(), formattedValue); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, List.of(record), null); } } - } - - - /******************************************************************************* - ** For a list of records, set their display values - *******************************************************************************/ - public static void setDisplayValuesInRecord(Map fields, QRecord record) - { for(Map.Entry entry : fields.entrySet()) { String fieldName = entry.getKey(); @@ -490,6 +479,13 @@ public class QValueFormatter { adornmentValues = fileDownloadAdornment.get().getValues(); } + else + { + /////////////////////////////////////////////////////// + // don't change blobs unless they are file-downloads // + /////////////////////////////////////////////////////// + continue; + } String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD)); String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index 7816dba4..9657b2fb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -275,8 +275,19 @@ public class SearchPossibleValueSourceAction queryInput.setFilter(queryFilter); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - List ids = queryOutput.getRecords().stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList(); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + String fieldName; + if(StringUtils.hasContent(possibleValueSource.getOverrideIdField())) + { + fieldName = possibleValueSource.getOverrideIdField(); + } + else + { + fieldName = table.getPrimaryKeyField(); + } + + List ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList(); List> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, ids); output.setResults(qPossibleValues); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java index 778df8f2..0338e34b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java @@ -27,6 +27,7 @@ import java.util.Set; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -44,7 +45,8 @@ public class ValueBehaviorApplier public enum Action { INSERT, - UPDATE + UPDATE, + FORMATTING } @@ -63,7 +65,34 @@ public class ValueBehaviorApplier { for(FieldBehavior fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors())) { - fieldBehavior.apply(action, recordList, instance, table, field, behaviorsToOmit); + boolean applyBehavior = true; + if(behaviorsToOmit != null && behaviorsToOmit.contains(fieldBehavior)) + { + ///////////////////////////////////////////////////////////////////////////////////////// + // if we're given a set of behaviors to omit, and this behavior is in there, then skip // + ///////////////////////////////////////////////////////////////////////////////////////// + applyBehavior = false; + } + + if(Action.FORMATTING == action && !(fieldBehavior instanceof FieldDisplayBehavior)) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // for the formatting action, do not apply the behavior unless it is a field-display-behavior // + //////////////////////////////////////////////////////////////////////////////////////////////// + applyBehavior = false; + } + else if(Action.FORMATTING != action && fieldBehavior instanceof FieldDisplayBehavior) + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // for non-formatting actions, do not apply the behavior IF it is a field-display-behavior // + ///////////////////////////////////////////////////////////////////////////////////////////// + applyBehavior = false; + } + + if(applyBehavior) + { + fieldBehavior.apply(action, recordList, instance, table, field); + } } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java index 1d1a52bb..7cf1aea6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java @@ -34,5 +34,11 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; *******************************************************************************/ public record CapturedContext(QInstance qInstance, QSession qSession, QBackendTransaction qBackendTransaction, Stack actionStack) { - + /******************************************************************************* + ** Simpler constructor + *******************************************************************************/ + public CapturedContext(QInstance qInstance, QSession qSession) + { + this(qInstance, qSession, null, null); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java index 69cb0dbd..cfc931b4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeVoidVoidMethod; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -54,6 +55,7 @@ public class QContext private static ThreadLocal> objectsThreadLocal = new ThreadLocal<>(); + /******************************************************************************* ** private constructor - class is not meant to be instantiated. *******************************************************************************/ @@ -105,6 +107,25 @@ public class QContext + /******************************************************************************* + ** + *******************************************************************************/ + public static void withTemporaryContext(CapturedContext context, UnsafeVoidVoidMethod method) throws T + { + CapturedContext originalContext = QContext.capture(); + try + { + QContext.init(context); + method.run(); + } + finally + { + QContext.init(originalContext); + } + } + + + /******************************************************************************* ** Init a new thread with the context captured from a different thread. e.g., ** when starting some async task. @@ -267,6 +288,7 @@ public class QContext } + /******************************************************************************* ** get one named object from the Context for the current thread. may return null. *******************************************************************************/ @@ -280,6 +302,7 @@ public class QContext } + /******************************************************************************* ** get one named object from the Context for the current thread, cast to the ** specified type if possible. if not found, or wrong type, empty is returned. diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 232ade6c..a87713e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -77,6 +77,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -159,6 +160,18 @@ public class QInstanceEnricher } enrichJoins(); + + ////////////////////////////////////////////////////////////////////////////// + // if the instance DOES have 1 or more scheduler, but no schedulable types, // + // then go ahead and add the default set that qqq knows about // + ////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(qInstance.getSchedulers())) + { + if(CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulableTypes())) + { + QScheduleManager.defineDefaultSchedulableTypesInInstance(qInstance); + } + } } @@ -371,9 +384,9 @@ public class QInstanceEnricher process.setLabel(nameToLabel(process.getName())); } - if(process.getStepList() != null) + for(QStepMetaData step : CollectionUtils.nonNullMap(process.getAllSteps()).values()) { - process.getStepList().forEach(this::enrichStep); + enrichStep(step); } for(QSupplementalProcessMetaData supplementalProcessMetaData : CollectionUtils.nonNullMap(process.getSupplementalMetaData()).values()) @@ -1030,6 +1043,50 @@ public class QInstanceEnricher + /******************************************************************************* + ** Do a default mapping from an underscore_style field name to a camelCase name. + ** + ** Examples: + **
    + **
  • word_another_word_more_words -> wordAnotherWordMoreWords
    • + **
  • l_ul_ul_ul -> lUlUlUl
    • + **
  • tla_first -> tlaFirst
    • + **
  • word_then_tla_in_middle -> wordThenTlaInMiddle
    • + **
  • end_with_tla -> endWithTla
    • + **
  • tla_and_another_tla -> tlaAndAnotherTla
    • + **
  • ALL_CAPS -> allCaps
    • + **
+ *******************************************************************************/ + public static String inferNameFromBackendName(String backendName) + { + StringBuilder rs = new StringBuilder(); + + //////////////////////////////////////////////////////////////////////////////////////// + // build a list of words in the name, then join them with _ and lower-case the result // + //////////////////////////////////////////////////////////////////////////////////////// + String[] words = backendName.toLowerCase(Locale.ROOT).split("_"); + for(int i = 0; i < words.length; i++) + { + String word = words[i]; + if(i == 0) + { + rs.append(word); + } + else + { + rs.append(word.substring(0, 1).toUpperCase()); + if(word.length() > 1) + { + rs.append(word.substring(1)); + } + } + } + + return (rs.toString()); + } + + + /******************************************************************************* ** If a app didn't have any sections, generate "sensible defaults" *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java index f0e6968c..ed5d69da 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java @@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.help.HelpFormat; import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -111,6 +112,7 @@ public class QInstanceHelpContentManager String processName = nameValuePairs.get("process"); String fieldName = nameValuePairs.get("field"); String sectionName = nameValuePairs.get("section"); + String stepName = nameValuePairs.get("step"); String widgetName = nameValuePairs.get("widget"); String slotName = nameValuePairs.get("slot"); @@ -145,12 +147,11 @@ public class QInstanceHelpContentManager } else if(StringUtils.hasContent(processName)) { - processHelpContentForProcess(key, processName, fieldName, roles, helpContent); + processHelpContentForProcess(key, processName, fieldName, stepName, roles, helpContent); } else if(StringUtils.hasContent(widgetName)) { - processHelpContentForWidget(key, widgetName, slotName, helpContent); - + processHelpContentForWidget(key, widgetName, slotName, roles, helpContent); } } catch(Exception e) @@ -209,6 +210,10 @@ public class QInstanceHelpContentManager optionalSection.get().removeHelpContent(roles); } } + else + { + LOG.info("Unrecognized key format for table help content", logPair("key", key)); + } } @@ -216,7 +221,7 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForProcess(String key, String processName, String fieldName, Set roles, QHelpContent helpContent) + private static void processHelpContentForProcess(String key, String processName, String fieldName, String stepName, Set roles, QHelpContent helpContent) { QProcessMetaData process = QContext.getQInstance().getProcess(processName); if(process == null) @@ -245,6 +250,30 @@ public class QInstanceHelpContentManager optionalField.get().removeHelpContent(roles); } } + else if(StringUtils.hasContent(stepName)) + { + ///////////////////////////// + // handle a process screen // + ///////////////////////////// + QFrontendStepMetaData frontendStep = process.getFrontendStep(stepName); + + if(frontendStep == null) + { + LOG.info("Unrecognized process step in help content", logPair("key", key)); + } + else if(helpContent != null) + { + frontendStep.withHelpContent(helpContent); + } + else + { + frontendStep.removeHelpContent(roles); + } + } + else + { + LOG.info("Unrecognized key format for process help content", logPair("key", key)); + } } @@ -252,7 +281,7 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForWidget(String key, String widgetName, String slotName, QHelpContent helpContent) + private static void processHelpContentForWidget(String key, String widgetName, String slotName, Set roles, QHelpContent helpContent) { QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName); if(!StringUtils.hasContent(slotName)) @@ -265,22 +294,14 @@ public class QInstanceHelpContentManager } else { - Map widgetHelpContent = widget.getHelpContent(); - if(widgetHelpContent == null) - { - widgetHelpContent = new HashMap<>(); - } - if(helpContent != null) { - widgetHelpContent.put(slotName, helpContent); + widget.withHelpContent(slotName, helpContent); } else { - widgetHelpContent.remove(slotName); + widget.removeHelpContent(slotName, roles); } - - widget.setHelpContent(widgetHelpContent); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index f16c4d7f..765e8e7c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -35,6 +36,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; import java.util.function.Supplier; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; @@ -62,6 +64,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaD import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; @@ -83,6 +86,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; @@ -104,6 +108,8 @@ import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; +import org.quartz.CronExpression; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -146,6 +152,7 @@ public class QInstanceValidator // once, during the enrichment/validation work, so, capture it, and store it back in the instance. // ///////////////////////////////////////////////////////////////////////////////////////////////////// JoinGraph joinGraph = null; + long start = System.currentTimeMillis(); try { ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -186,6 +193,9 @@ public class QInstanceValidator validateUniqueTopLevelNames(qInstance); runPlugins(QInstance.class, qInstance, qInstance); + + long end = System.currentTimeMillis(); + LOG.info("Validation (and enrichment) performance", logPair("millis", (end - start))); } catch(Exception e) { @@ -204,6 +214,17 @@ public class QInstanceValidator + /******************************************************************************* + ** + *******************************************************************************/ + public void revalidate(QInstance qInstance) throws QInstanceValidationException + { + qInstance.setHasBeenValidated(null); + validate(qInstance); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -432,6 +453,11 @@ public class QInstanceValidator assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name); } + if(queue.getSchedule() != null) + { + validateScheduleMetaData(queue.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: "); + } + runPlugins(QQueueMetaData.class, queue, qInstance); }); } @@ -615,6 +641,11 @@ public class QInstanceValidator supplementalTableMetaData.validate(qInstance, table, this); } + if(table.getShareableTableMetaData() != null) + { + table.getShareableTableMetaData().validate(qInstance, table, this); + } + runPlugins(QTableMetaData.class, table, qInstance); }); } @@ -642,17 +673,20 @@ public class QInstanceValidator { if(assertCondition(CollectionUtils.nullSafeHasContents(exposedJoin.getJoinPath()), joinPrefix + "is missing a joinPath.")) { - joinConnectionsForTable = Objects.requireNonNullElseGet(joinConnectionsForTable, () -> joinGraph.getJoinConnections(table.getName())); - - boolean foundJoinConnection = false; - for(JoinGraph.JoinConnectionList joinConnectionList : joinConnectionsForTable) + if(joinGraph != null) { - if(joinConnectionList.matchesJoinPath(exposedJoin.getJoinPath())) + joinConnectionsForTable = Objects.requireNonNullElseGet(joinConnectionsForTable, () -> joinGraph.getJoinConnections(table.getName())); + + boolean foundJoinConnection = false; + for(JoinGraph.JoinConnectionList joinConnectionList : joinConnectionsForTable) { - foundJoinConnection = true; + if(joinConnectionList.matchesJoinPath(exposedJoin.getJoinPath())) + { + foundJoinConnection = true; + } } + assertCondition(foundJoinConnection, joinPrefix + "specified a joinPath [" + exposedJoin.getJoinPath() + "] which does not match a valid join connection in the instance."); } - assertCondition(foundJoinConnection, joinPrefix + "specified a joinPath [" + exposedJoin.getJoinPath() + "] which does not match a valid join connection in the instance."); assertCondition(!usedJoinPaths.contains(exposedJoin.getJoinPath()), tablePrefix + "has more than one join with the joinPath: " + exposedJoin.getJoinPath()); usedJoinPaths.add(exposedJoin.getJoinPath()); @@ -702,7 +736,6 @@ public class QInstanceValidator { String prefix = "Table " + table.getName() + " "; - RECORD_SECURITY_LOCKS_LOOP: for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks())) { if(!assertCondition(recordSecurityLock != null, prefix + "has a null recordSecurityLock (did you mean to give it a null list of locks?)")) @@ -710,90 +743,14 @@ public class QInstanceValidator continue; } - String securityKeyTypeName = recordSecurityLock.getSecurityKeyType(); - if(assertCondition(StringUtils.hasContent(securityKeyTypeName), prefix + "has a recordSecurityLock that is missing a securityKeyType")) + if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock) { - assertCondition(qInstance.getSecurityKeyType(securityKeyTypeName) != null, prefix + "has a recordSecurityLock with an unrecognized securityKeyType: " + securityKeyTypeName); + validateMultiRecordSecurityLock(qInstance, table, multiRecordSecurityLock, prefix); } - - prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") "; - - assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope"); - - boolean hasAnyBadJoins = false; - for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain())) + else { - if(!assertCondition(qInstance.getJoin(joinName) != null, prefix + "has an unrecognized joinName: " + joinName)) - { - hasAnyBadJoins = true; - } + validateRecordSecurityLock(qInstance, table, recordSecurityLock, prefix); } - - String fieldName = recordSecurityLock.getFieldName(); - - //////////////////////////////////////////////////////////////////////////////// - // don't bother trying to validate field names if we know we have a bad join. // - //////////////////////////////////////////////////////////////////////////////// - if(assertCondition(StringUtils.hasContent(fieldName), prefix + "is missing a fieldName") && !hasAnyBadJoins) - { - if(fieldName.contains(".")) - { - if(assertCondition(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " looks like a join (has a dot), but no joinNameChain was given.")) - { - List joins = new ArrayList<>(); - - /////////////////////////////////////////////////////////////////////////////////////////////////// - // ok - so - the join name chain is going to be like this: // - // for a table: orderLineItemExtrinsic (that's 2 away from order, where the security field is): // - // - securityFieldName = order.clientId // - // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic // - // so - to navigate from the table to the security field, we need to reverse the joinNameChain, // - // and step (via tmpTable variable) back to the securityField // - /////////////////////////////////////////////////////////////////////////////////////////////////// - ArrayList joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain())); - Collections.reverse(joinNameChain); - - QTableMetaData tmpTable = table; - - for(String joinName : joinNameChain) - { - QJoinMetaData join = qInstance.getJoin(joinName); - if(join == null) - { - errors.add(prefix + "joinNameChain contained an unrecognized join: " + joinName); - continue RECORD_SECURITY_LOCKS_LOOP; - } - - if(join.getLeftTable().equals(tmpTable.getName())) - { - joins.add(new QueryJoin(join)); - tmpTable = qInstance.getTable(join.getRightTable()); - } - else if(join.getRightTable().equals(tmpTable.getName())) - { - joins.add(new QueryJoin(join.flip())); - tmpTable = qInstance.getTable(join.getLeftTable()); - } - else - { - errors.add(prefix + "joinNameChain could not be followed through join: " + joinName); - continue RECORD_SECURITY_LOCKS_LOOP; - } - } - - assertCondition(findField(qInstance, table, joins, fieldName), prefix + "has an unrecognized fieldName: " + fieldName); - } - } - else - { - if(assertCondition(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " does not look like a join (does not have a dot), but a joinNameChain was given.")) - { - assertNoException(() -> table.getField(fieldName), prefix + "has an unrecognized fieldName: " + fieldName); - } - } - } - - assertCondition(recordSecurityLock.getNullValueBehavior() != null, prefix + "is missing a nullValueBehavior"); } } @@ -802,7 +759,121 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field) + private void validateMultiRecordSecurityLock(QInstance qInstance, QTableMetaData table, MultiRecordSecurityLock multiRecordSecurityLock, String prefix) + { + assertCondition(multiRecordSecurityLock.getOperator() != null, prefix + "has a MultiRecordSecurityLock that is missing an operator"); + + for(RecordSecurityLock lock : multiRecordSecurityLock.getLocks()) + { + validateRecordSecurityLock(qInstance, table, lock, prefix); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void validateRecordSecurityLock(QInstance qInstance, QTableMetaData table, RecordSecurityLock recordSecurityLock, String prefix) + { + String securityKeyTypeName = recordSecurityLock.getSecurityKeyType(); + if(assertCondition(StringUtils.hasContent(securityKeyTypeName), prefix + "has a recordSecurityLock that is missing a securityKeyType")) + { + assertCondition(qInstance.getSecurityKeyType(securityKeyTypeName) != null, prefix + "has a recordSecurityLock with an unrecognized securityKeyType: " + securityKeyTypeName); + } + + prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") "; + + assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope"); + + boolean hasAnyBadJoins = false; + for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain())) + { + if(!assertCondition(qInstance.getJoin(joinName) != null, prefix + "has an unrecognized joinName: " + joinName)) + { + hasAnyBadJoins = true; + } + } + + String fieldName = recordSecurityLock.getFieldName(); + + //////////////////////////////////////////////////////////////////////////////// + // don't bother trying to validate field names if we know we have a bad join. // + //////////////////////////////////////////////////////////////////////////////// + if(assertCondition(StringUtils.hasContent(fieldName), prefix + "is missing a fieldName") && !hasAnyBadJoins) + { + if(fieldName.contains(".")) + { + if(assertCondition(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " looks like a join (has a dot), but no joinNameChain was given.")) + { + String[] split = fieldName.split("\\."); + String joinTableName = split[0]; + String joinFieldName = split[1]; + + List joins = new ArrayList<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // ok - so - the join name chain is going to be like this: // + // for a table: orderLineItemExtrinsic (that's 2 away from order, where the security field is): // + // - securityFieldName = order.clientId // + // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic // + // so - to navigate from the table to the security field, we need to reverse the joinNameChain, // + // and step (via tmpTable variable) back to the securityField // + /////////////////////////////////////////////////////////////////////////////////////////////////// + ArrayList joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain())); + Collections.reverse(joinNameChain); + + QTableMetaData tmpTable = table; + + for(String joinName : joinNameChain) + { + QJoinMetaData join = qInstance.getJoin(joinName); + if(join == null) + { + errors.add(prefix + "joinNameChain contained an unrecognized join: " + joinName); + return; + } + + if(join.getLeftTable().equals(tmpTable.getName())) + { + joins.add(new QueryJoin(join)); + tmpTable = qInstance.getTable(join.getRightTable()); + } + else if(join.getRightTable().equals(tmpTable.getName())) + { + joins.add(new QueryJoin(join.flip())); + tmpTable = qInstance.getTable(join.getLeftTable()); + } + else + { + errors.add(prefix + "joinNameChain could not be followed through join: " + joinName); + return; + } + } + + assertCondition(Objects.equals(tmpTable.getName(), joinTableName), prefix + "has a joinNameChain doesn't end in the expected table [" + joinTableName + "] (was: " + tmpTable.getName() + ")"); + + assertCondition(findField(qInstance, table, joins, fieldName), prefix + "has an unrecognized fieldName: " + fieldName); + } + } + else + { + if(assertCondition(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " does not look like a join (does not have a dot), but a joinNameChain was given.")) + { + assertNoException(() -> table.getField(fieldName), prefix + "has an unrecognized fieldName: " + fieldName); + } + } + } + + assertCondition(recordSecurityLock.getNullValueBehavior() != null, prefix + "is missing a nullValueBehavior"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private > void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field) { assertCondition(Objects.equals(fieldName, field.getName()), "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + "."); @@ -815,12 +886,32 @@ public class QInstanceValidator String prefix = "Field " + fieldName + " in table " + tableName + " "; + /////////////////////////////////////////////////// + // validate things we know about field behaviors // + /////////////////////////////////////////////////// ValueTooLongBehavior behavior = field.getBehaviorOrDefault(qInstance, ValueTooLongBehavior.class); if(behavior != null && !behavior.equals(ValueTooLongBehavior.PASS_THROUGH)) { assertCondition(field.getMaxLength() != null, prefix + "specifies a ValueTooLongBehavior, but not a maxLength."); } + Set>> usedFieldBehaviorTypes = new HashSet<>(); + if(field.getBehaviors() != null) + { + for(FieldBehavior fieldBehavior : field.getBehaviors()) + { + Class> behaviorClass = (Class>) fieldBehavior.getClass(); + + errors.addAll(fieldBehavior.validateBehaviorConfiguration(table, field)); + + if(!fieldBehavior.allowMultipleBehaviorsOfThisType()) + { + assertCondition(!usedFieldBehaviorTypes.contains(behaviorClass), prefix + "has more than 1 fieldBehavior of type " + behaviorClass.getSimpleName() + ", which is not allowed for this type"); + } + usedFieldBehaviorTypes.add(behaviorClass); + } + } + if(field.getMaxLength() != null) { assertCondition(field.getMaxLength() > 0, prefix + "has an invalid maxLength (" + field.getMaxLength() + ") - must be greater than 0."); @@ -1013,6 +1104,11 @@ public class QInstanceValidator assertCondition(qInstance.getAutomationProvider(providerName) != null, " has an unrecognized providerName: " + providerName); } + if(automationDetails.getSchedule() != null) + { + validateScheduleMetaData(automationDetails.getSchedule(), qInstance, prefix + " automationDetails, schedule: "); + } + ////////////////////////////////// // validate the status tracking // ////////////////////////////////// @@ -1349,12 +1445,17 @@ public class QInstanceValidator /////////////////////////////////// // validate steps in the process // /////////////////////////////////// + Set usedStepNames = new HashSet<>(); if(assertCondition(CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + ".")) { int index = 0; for(QStepMetaData step : process.getStepList()) { - assertCondition(StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName); + if(assertCondition(StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName)) + { + assertCondition(!usedStepNames.contains(step.getName()), "Duplicate step name [" + step.getName() + "] in process " + processName); + usedStepNames.add(step.getName()); + } index++; //////////////////////////////////////////// @@ -1386,7 +1487,7 @@ public class QInstanceValidator warn("Error loading expectedType for field [" + fieldMetaData.getName() + "] in process [" + processName + "]: " + e.getMessage()); } - validateSimpleCodeReference("Process " + processName + " code reference: ", codeReference, expectedClass); + validateSimpleCodeReference("Process " + processName + " code reference:", codeReference, expectedClass); } } } @@ -1394,19 +1495,35 @@ public class QInstanceValidator } } + if(process.getCancelStep() != null) + { + if(assertCondition(process.getCancelStep().getCode() != null, "Cancel step is missing a code reference, in process " + processName)) + { + validateSimpleCodeReference("Process " + processName + " cancel step code reference: ", process.getCancelStep().getCode(), BackendStep.class); + } + } + /////////////////////////////////////////////////////////////////////////////// // if the process has a schedule, make sure required schedule data populated // /////////////////////////////////////////////////////////////////////////////// if(process.getSchedule() != null) { QScheduleMetaData schedule = process.getSchedule(); - assertCondition(schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null, "Either repeat millis or repeat seconds must be set on schedule in process " + processName); + validateScheduleMetaData(schedule, qInstance, "Process " + processName + ", schedule: "); + } - if(schedule.getVariantBackend() != null) + if(process.getVariantBackend() != null) + { + if(qInstance.getBackends() != null) { - assertCondition(qInstance.getBackend(schedule.getVariantBackend()) != null, "A variant backend was not found for " + schedule.getVariantBackend()); - assertCondition(schedule.getVariantRunStrategy() != null, "A variant run strategy was not set for " + schedule.getVariantBackend() + " on schedule in process " + processName); + assertCondition(qInstance.getBackend(process.getVariantBackend()) != null, "Process " + processName + ", a variant backend was not found named " + process.getVariantBackend()); } + + assertCondition(process.getVariantRunStrategy() != null, "A variant run strategy was not set for process " + processName + " (which does specify a variant backend)"); + } + else + { + assertCondition(process.getVariantRunStrategy() == null, "A variant run strategy was set for process " + processName + " (which isn't allowed, since it does not specify a variant backend)"); } for(QSupplementalProcessMetaData supplementalProcessMetaData : CollectionUtils.nonNullMap(process.getSupplementalMetaData()).values()) @@ -1421,6 +1538,50 @@ public class QInstanceValidator + /******************************************************************************* + ** + *******************************************************************************/ + private void validateScheduleMetaData(QScheduleMetaData schedule, QInstance qInstance, String prefix) + { + boolean isRepeat = schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null; + boolean isCron = StringUtils.hasContent(schedule.getCronExpression()); + assertCondition(isRepeat || isCron, prefix + " either repeatMillis or repeatSeconds or cronExpression must be set"); + assertCondition(!(isRepeat && isCron), prefix + " both a repeat time and cronExpression may not be set"); + + if(isCron) + { + boolean hasDelay = schedule.getInitialDelayMillis() != null || schedule.getInitialDelaySeconds() != null; + assertCondition(!hasDelay, prefix + " a cron schedule may not have an initial delay"); + + try + { + CronExpression.validateExpression(schedule.getCronExpression()); + } + catch(ParseException pe) + { + errors.add(prefix + " invalid cron expression: " + pe.getMessage()); + } + + if(assertCondition(StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a cron schedule must specify a cronTimeZoneId")) + { + String[] availableIDs = TimeZone.getAvailableIDs(); + Optional first = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst(); + assertCondition(first.isPresent(), prefix + " unrecognized cronTimeZoneId: " + schedule.getCronTimeZoneId()); + } + } + else + { + assertCondition(!StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a non-cron schedule must not specify a cronTimeZoneId"); + } + + if(assertCondition(StringUtils.hasContent(schedule.getSchedulerName()), prefix + " is missing a scheduler name")) + { + assertCondition(qInstance.getScheduler(schedule.getSchedulerName()) != null, prefix + " is referencing an unknown scheduler name: " + schedule.getSchedulerName()); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java new file mode 100644 index 00000000..635a7410 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java @@ -0,0 +1,164 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.logging; + + +import org.apache.logging.log4j.Level; +import org.json.JSONException; +import org.json.JSONObject; + + +/******************************************************************************* + ** A log message, which can be "collected" by the QCollectingLogger. + *******************************************************************************/ +public class CollectedLogMessage +{ + private Level level; + private String message; + private Throwable exception; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public CollectedLogMessage() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "CollectedLogMessage{level=" + level + ", message='" + message + '\'' + ", exception=" + exception + '}'; + } + + + + /******************************************************************************* + ** Getter for message + *******************************************************************************/ + public String getMessage() + { + return (this.message); + } + + + + /******************************************************************************* + ** Setter for message + *******************************************************************************/ + public void setMessage(String message) + { + this.message = message; + } + + + + /******************************************************************************* + ** Fluent setter for message + *******************************************************************************/ + public CollectedLogMessage withMessage(String message) + { + this.message = message; + return (this); + } + + + + /******************************************************************************* + ** Getter for exception + *******************************************************************************/ + public Throwable getException() + { + return (this.exception); + } + + + + /******************************************************************************* + ** Setter for exception + *******************************************************************************/ + public void setException(Throwable exception) + { + this.exception = exception; + } + + + + /******************************************************************************* + ** Fluent setter for exception + *******************************************************************************/ + public CollectedLogMessage withException(Throwable exception) + { + this.exception = exception; + return (this); + } + + + + /******************************************************************************* + ** Getter for level + ** + *******************************************************************************/ + public Level getLevel() + { + return level; + } + + + + /******************************************************************************* + ** Setter for level + ** + *******************************************************************************/ + public void setLevel(Level level) + { + this.level = level; + } + + + + /******************************************************************************* + ** Fluent setter for level + ** + *******************************************************************************/ + public CollectedLogMessage withLevel(Level level) + { + this.level = level; + return (this); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public JSONObject getMessageAsJSONObject() throws JSONException + { + return (new JSONObject(getMessage())); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java index fd654a0f..6a79a874 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java @@ -79,7 +79,7 @@ public class LogPair } else if(value instanceof LogPair[] subLogPairs) { - String subLogPairsString = Arrays.stream(subLogPairs).map(LogPair::toString).collect(Collectors.joining(",")); + String subLogPairsString = Arrays.stream(subLogPairs).filter(Objects::nonNull).map(LogPair::toString).collect(Collectors.joining(",")); valueString = '{' + subLogPairsString + '}'; } else if(value instanceof UnsafeSupplier us) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java new file mode 100644 index 00000000..b10d3c07 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java @@ -0,0 +1,155 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.logging; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ObjectMessage; +import org.apache.logging.log4j.message.SimpleMessageFactory; +import org.apache.logging.log4j.simple.SimpleLogger; +import org.apache.logging.log4j.util.PropertiesUtil; + + +/******************************************************************************* + ** QQQ log4j implementation, used within a QLogger, to "collect" log messages + ** in an internal list - the idea being - for tests, to assert that logs happened. + *******************************************************************************/ +public class QCollectingLogger extends SimpleLogger +{ + private List collectedMessages = new ArrayList<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // just in case one of these gets activated, and left on, put a limit on how many messages we'll collect // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + private int capacity = 100; + + private Logger logger; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QCollectingLogger(Logger logger) + { + super(logger.getName(), logger.getLevel(), false, false, true, false, "", new SimpleMessageFactory(), new PropertiesUtil(new Properties()), System.out); + this.logger = logger; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void logMessage(String fqcn, Level level, Marker marker, Message message, Throwable throwable) + { + //////////////////////////////////////////// + // add this log message to our collection // + //////////////////////////////////////////// + collectedMessages.add(new CollectedLogMessage() + .withLevel(level) + .withMessage(message.getFormattedMessage()) + .withException(throwable)); + + //////////////////////////////////////////////////////////////////////////////////////// + // if we've gone over our capacity, remove the 1st entry until we're back at capacity // + //////////////////////////////////////////////////////////////////////////////////////// + while(collectedMessages.size() > capacity) + { + collectedMessages.remove(0); + } + + ////////////////////////////////////////////////////////////////////// + // update the message that we log to indicate that we collected it. // + // if it looks like JSON, insert as a name:value pair; else text. // + ////////////////////////////////////////////////////////////////////// + String formattedMessage = message.getFormattedMessage(); + String updatedMessage; + if(formattedMessage.startsWith("{")) + { + updatedMessage = """ + {"collected":true,""" + formattedMessage.substring(1); + } + else + { + updatedMessage = "[Collected] " + formattedMessage; + } + ObjectMessage myMessage = new ObjectMessage(updatedMessage); + + /////////////////////////////////////////////////////////////////////////////////////// + // log the message with the original log4j logger, with our slightly updated message // + /////////////////////////////////////////////////////////////////////////////////////// + logger.logMessage(level, marker, fqcn, null, myMessage, throwable); + } + + + + /******************************************************************************* + ** Setter for logger + ** + *******************************************************************************/ + public void setLogger(Logger logger) + { + this.logger = logger; + } + + + + /******************************************************************************* + ** Getter for collectedMessages + ** + *******************************************************************************/ + public List getCollectedMessages() + { + return collectedMessages; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void clear() + { + this.collectedMessages.clear(); + } + + + + /******************************************************************************* + ** Setter for capacity + ** + *******************************************************************************/ + public void setCapacity(int capacity) + { + this.capacity = capacity; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java index 3683c62a..42bfe93d 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.logging; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -119,6 +120,56 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public static QCollectingLogger activateCollectingLoggerForClass(Class c) + { + Logger loggerFromLogManager = LogManager.getLogger(c); + QCollectingLogger collectingLogger = new QCollectingLogger(loggerFromLogManager); + + QLogger qLogger = getLogger(c); + qLogger.setLogger(collectingLogger); + + return collectingLogger; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void deactivateCollectingLoggerForClass(Class c) + { + Logger loggerFromLogManager = LogManager.getLogger(c); + QLogger qLogger = getLogger(c); + qLogger.setLogger(loggerFromLogManager); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public T warnAndThrow(T t, LogPair... logPairs) throws T + { + warn(t.getMessage(), t, logPairs); + throw (t); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public T errorAndThrow(T t, LogPair... logPairs) throws T + { + error(t.getMessage(), t, logPairs); + throw (t); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -518,7 +569,7 @@ public class QLogger /******************************************************************************* ** *******************************************************************************/ - private String makeJsonString(String message, Throwable t, List logPairList) + protected String makeJsonString(String message, Throwable t, List logPairList) { if(logPairList == null) { @@ -567,7 +618,10 @@ public class QLogger { user = session.getUser().getIdReference(); } - sessionLogPair = logPair("session", logPair("id", session.getUuid()), logPair("user", user)); + + LogPair variantsLogPair = getVariantsLogPair(session); + + sessionLogPair = logPair("session", logPair("id", session.getUuid()), logPair("user", user), variantsLogPair); } try @@ -587,6 +641,38 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + private static LogPair getVariantsLogPair(QSession session) + { + LogPair variantsLogPair = null; + try + { + if(session.getBackendVariants() != null) + { + LogPair[] variants = new LogPair[session.getBackendVariants().size()]; + + int i = 0; + for(Map.Entry entry : session.getBackendVariants().entrySet()) + { + variants[i] = new LogPair(entry.getKey(), entry.getValue()); + } + + variantsLogPair = new LogPair("variants", variants); + } + } + catch(Exception e) + { + //////////////// + // leave null // + //////////////// + } + return variantsLogPair; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -620,4 +706,15 @@ public class QLogger exceptionList.get(0).setHasLoggedLevel(level); return (level); } + + + + /******************************************************************************* + ** Setter for logger + ** + *******************************************************************************/ + private void setLogger(Logger logger) + { + this.logger = logger; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java index 45bd2ca6..9faf3dc4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java @@ -23,8 +23,8 @@ package com.kingsrook.qqq.backend.core.model; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; /******************************************************************************* @@ -42,7 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; ** implement this interface. or, same idea for a QRecordEntity that provides ** its own TableMetaData. *******************************************************************************/ -public interface MetaDataProducerInterface +public interface MetaDataProducerInterface { int DEFAULT_SORT_ORDER = 500; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java index 429948d4..3c98715b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java @@ -26,6 +26,7 @@ import java.util.UUID; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; +import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; @@ -139,7 +140,7 @@ public class AbstractActionInput //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // don't return null here (too easy to NPE). instead, if someone wants one of these, create one and give it to them. // //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - asyncJobCallback = new AsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus()); + asyncJobCallback = new NonPersistedAsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus().withJobName(getClass().getSimpleName())); } return asyncJobCallback; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Attachment.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Attachment.java new file mode 100644 index 00000000..f3720fac --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Attachment.java @@ -0,0 +1,95 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class Attachment +{ + private byte[] contents; + private String name; + + + + /******************************************************************************* + ** Getter for contents + *******************************************************************************/ + public byte[] getContents() + { + return (this.contents); + } + + + + /******************************************************************************* + ** Setter for contents + *******************************************************************************/ + public void setContents(byte[] contents) + { + this.contents = contents; + } + + + + /******************************************************************************* + ** Fluent setter for contents + *******************************************************************************/ + public Attachment withContents(byte[] contents) + { + this.contents = contents; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public Attachment withName(String name) + { + this.name = name; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Content.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Content.java new file mode 100644 index 00000000..07d9cab0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Content.java @@ -0,0 +1,95 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class Content +{ + private String body; + private ContentRole contentRole; + + + + /******************************************************************************* + ** Getter for body + *******************************************************************************/ + public String getBody() + { + return (this.body); + } + + + + /******************************************************************************* + ** Setter for body + *******************************************************************************/ + public void setBody(String body) + { + this.body = body; + } + + + + /******************************************************************************* + ** Fluent setter for body + *******************************************************************************/ + public Content withBody(String body) + { + this.body = body; + return (this); + } + + + + /******************************************************************************* + ** Getter for contentRole + *******************************************************************************/ + public ContentRole getContentRole() + { + return (this.contentRole); + } + + + + /******************************************************************************* + ** Setter for contentRole + *******************************************************************************/ + public void setContentRole(ContentRole contentRole) + { + this.contentRole = contentRole; + } + + + + /******************************************************************************* + ** Fluent setter for contentRole + *******************************************************************************/ + public Content withContentRole(ContentRole contentRole) + { + this.contentRole = contentRole; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/ContentRole.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/ContentRole.java new file mode 100644 index 00000000..d18dd03f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/ContentRole.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface ContentRole +{ + + /******************************************************************************* + ** + *******************************************************************************/ + enum Default implements ContentRole + { + DEFAULT + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/MultiParty.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/MultiParty.java new file mode 100644 index 00000000..f736b3e7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/MultiParty.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging; + + +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MultiParty extends Party +{ + private List partyList; + + + + /******************************************************************************* + ** Getter for partyList + *******************************************************************************/ + public List getPartyList() + { + return (this.partyList); + } + + + + /******************************************************************************* + ** Setter for partyList + *******************************************************************************/ + public void setPartyList(List partyList) + { + this.partyList = partyList; + } + + + + /******************************************************************************* + ** Fluent setter for partyList + *******************************************************************************/ + public MultiParty withPartyList(List partyList) + { + this.partyList = partyList; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MultiParty withParty(Party party) + { + addParty(party); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addParty(Party party) + { + if(this.partyList == null) + { + this.partyList = new ArrayList<>(); + } + this.partyList.add(party); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Party.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Party.java new file mode 100644 index 00000000..13b378ba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Party.java @@ -0,0 +1,127 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class Party +{ + private String label; + private String address; + private PartyRole role; + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public Party withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for address + *******************************************************************************/ + public String getAddress() + { + return (this.address); + } + + + + /******************************************************************************* + ** Setter for address + *******************************************************************************/ + public void setAddress(String address) + { + this.address = address; + } + + + + /******************************************************************************* + ** Fluent setter for address + *******************************************************************************/ + public Party withAddress(String address) + { + this.address = address; + return (this); + } + + + + /******************************************************************************* + ** Getter for role + *******************************************************************************/ + public PartyRole getRole() + { + return (this.role); + } + + + + /******************************************************************************* + ** Setter for role + *******************************************************************************/ + public void setRole(PartyRole role) + { + this.role = role; + } + + + + /******************************************************************************* + ** Fluent setter for role + *******************************************************************************/ + public Party withRole(PartyRole role) + { + this.role = role; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/PartyRole.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/PartyRole.java new file mode 100644 index 00000000..d70461b5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/PartyRole.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface PartyRole +{ + + /******************************************************************************* + ** + *******************************************************************************/ + enum Default implements PartyRole + { + DEFAULT + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/SendMessageInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/SendMessageInput.java new file mode 100644 index 00000000..f502dfa3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/SendMessageInput.java @@ -0,0 +1,278 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SendMessageInput extends AbstractActionInput +{ + private String messagingProviderName; + private Party to; + private Party from; + private String subject; + private List contentList; + private List attachmentList; + + + + /******************************************************************************* + ** Getter for to + *******************************************************************************/ + public Party getTo() + { + return (this.to); + } + + + + /******************************************************************************* + ** Setter for to + *******************************************************************************/ + public void setTo(Party to) + { + this.to = to; + } + + + + /******************************************************************************* + ** Fluent setter for to + *******************************************************************************/ + public SendMessageInput withTo(Party to) + { + this.to = to; + return (this); + } + + + + /******************************************************************************* + ** Getter for from + *******************************************************************************/ + public Party getFrom() + { + return (this.from); + } + + + + /******************************************************************************* + ** Setter for from + *******************************************************************************/ + public void setFrom(Party from) + { + this.from = from; + } + + + + /******************************************************************************* + ** Fluent setter for from + *******************************************************************************/ + public SendMessageInput withFrom(Party from) + { + this.from = from; + return (this); + } + + + + /******************************************************************************* + ** Getter for subject + *******************************************************************************/ + public String getSubject() + { + return (this.subject); + } + + + + /******************************************************************************* + ** Setter for subject + *******************************************************************************/ + public void setSubject(String subject) + { + this.subject = subject; + } + + + + /******************************************************************************* + ** Fluent setter for subject + *******************************************************************************/ + public SendMessageInput withSubject(String subject) + { + this.subject = subject; + return (this); + } + + + + /******************************************************************************* + ** Getter for contentList + *******************************************************************************/ + public List getContentList() + { + return (this.contentList); + } + + + + /******************************************************************************* + ** Setter for contentList + *******************************************************************************/ + public void setContentList(List contentList) + { + this.contentList = contentList; + } + + + + /******************************************************************************* + ** Fluent setter for contentList + *******************************************************************************/ + public SendMessageInput withContentList(List contentList) + { + this.contentList = contentList; + return (this); + } + + + + /******************************************************************************* + ** Getter for attachmentList + *******************************************************************************/ + public List getAttachmentList() + { + return (this.attachmentList); + } + + + + /******************************************************************************* + ** Setter for attachmentList + *******************************************************************************/ + public void setAttachmentList(List attachmentList) + { + this.attachmentList = attachmentList; + } + + + + /******************************************************************************* + ** Fluent setter for attachmentList + *******************************************************************************/ + public SendMessageInput withAttachmentList(List attachmentList) + { + this.attachmentList = attachmentList; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public SendMessageInput withContent(Content content) + { + addContent(content); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addContent(Content content) + { + if(this.contentList == null) + { + this.contentList = new ArrayList<>(); + } + this.contentList.add(content); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public SendMessageInput withAttachment(Attachment attachment) + { + addAttachment(attachment); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addAttachment(Attachment attachment) + { + if(this.attachmentList == null) + { + this.attachmentList = new ArrayList<>(); + } + this.attachmentList.add(attachment); + } + + + + /******************************************************************************* + ** Getter for messagingProviderName + *******************************************************************************/ + public String getMessagingProviderName() + { + return (this.messagingProviderName); + } + + + + /******************************************************************************* + ** Setter for messagingProviderName + *******************************************************************************/ + public void setMessagingProviderName(String messagingProviderName) + { + this.messagingProviderName = messagingProviderName; + } + + + + /******************************************************************************* + ** Fluent setter for messagingProviderName + *******************************************************************************/ + public SendMessageInput withMessagingProviderName(String messagingProviderName) + { + this.messagingProviderName = messagingProviderName; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/SendMessageOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/SendMessageOutput.java new file mode 100644 index 00000000..032064a7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/SendMessageOutput.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging; + + +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SendMessageOutput extends AbstractActionOutput +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/email/EmailContentRole.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/email/EmailContentRole.java new file mode 100644 index 00000000..9c8bf0d3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/email/EmailContentRole.java @@ -0,0 +1,35 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging.email; + + +import com.kingsrook.qqq.backend.core.model.actions.messaging.ContentRole; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum EmailContentRole implements ContentRole +{ + TEXT, + HTML +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/email/EmailPartyRole.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/email/EmailPartyRole.java new file mode 100644 index 00000000..4c7c8e6f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/email/EmailPartyRole.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.messaging.email; + + +import com.kingsrook.qqq.backend.core.model.actions.messaging.PartyRole; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum EmailPartyRole implements PartyRole +{ + TO, + CC, + BCC, + FROM, + REPLY_TO +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java index ccc95108..c418bd07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; /******************************************************************************* @@ -41,6 +42,11 @@ public class ProcessState implements Serializable private List stepList = new ArrayList<>(); private Optional nextStepName = Optional.empty(); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // maybe, remove this altogether - just let the frontend compute & send if needed... but how does it know last version...? // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + private List updatedFrontendStepList = null; + /******************************************************************************* @@ -139,4 +145,36 @@ public class ProcessState implements Serializable { this.stepList = stepList; } + + + + /******************************************************************************* + ** Getter for updatedFrontendStepList + *******************************************************************************/ + public List getUpdatedFrontendStepList() + { + return (this.updatedFrontendStepList); + } + + + + /******************************************************************************* + ** Setter for updatedFrontendStepList + *******************************************************************************/ + public void setUpdatedFrontendStepList(List updatedFrontendStepList) + { + this.updatedFrontendStepList = updatedFrontendStepList; + } + + + + /******************************************************************************* + ** Fluent setter for updatedFrontendStepList + *******************************************************************************/ + public ProcessState withUpdatedFrontendStepList(List updatedFrontendStepList) + { + this.updatedFrontendStepList = updatedFrontendStepList; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 18d63f75..bfaad833 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; +import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; @@ -450,7 +451,7 @@ public class RunBackendStepInput extends AbstractActionInput ///////////////////////////////////////////////////////////////////////// // avoid NPE in case we didn't have one of these! create a new one... // ///////////////////////////////////////////////////////////////////////// - asyncJobCallback = new AsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus()); + asyncJobCallback = new NonPersistedAsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus().withJobName(processName + "." + stepName)); } return (asyncJobCallback); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java index 9a915141..4754fcaa 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java @@ -27,10 +27,14 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -40,9 +44,13 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class RunBackendStepOutput extends AbstractActionOutput implements Serializable { + private String processName; + private ProcessState processState; private Exception exception; // todo - make optional + private String overrideLastStepName; // todo - does this need to go into state too?? + private List auditInputList = new ArrayList<>(); @@ -78,6 +86,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial public void seedFromRequest(RunBackendStepInput runBackendStepInput) { this.processState = runBackendStepInput.getProcessState(); + this.processName = runBackendStepInput.getProcessName(); } @@ -312,4 +321,111 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial auditInput.addAuditSingleInput(auditSingleInput); } + + + /******************************************************************************* + ** Getter for overrideLastStepName + *******************************************************************************/ + public String getOverrideLastStepName() + { + return (this.overrideLastStepName); + } + + + + /******************************************************************************* + ** Setter for overrideLastStepName + *******************************************************************************/ + public void setOverrideLastStepName(String overrideLastStepName) + { + this.overrideLastStepName = overrideLastStepName; + } + + + + /******************************************************************************* + ** Fluent setter for overrideLastStepName + *******************************************************************************/ + public RunBackendStepOutput withOverrideLastStepName(String overrideLastStepName) + { + this.overrideLastStepName = overrideLastStepName; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void updateStepList(List stepList) + { + getProcessState().setStepList(stepList); + + if(processName == null) + { + throw (new QRuntimeException("ProcessName was not set in this object, therefore updateStepList cannot complete successfully. Try to manually call setProcessName as a work around.")); + } + + QProcessMetaData processMetaData = QContext.getQInstance().getProcess(processName); + + ArrayList updatedFrontendStepList = new ArrayList<>(stepList.stream() + .map(name -> processMetaData.getStep(name)) + .filter(step -> step instanceof QFrontendStepMetaData) + .map(step -> (QFrontendStepMetaData) step) + .toList()); + + setUpdatedFrontendStepList(updatedFrontendStepList); + } + + + + /******************************************************************************* + ** Getter for processName + *******************************************************************************/ + public String getProcessName() + { + return (this.processName); + } + + + + /******************************************************************************* + ** Setter for processName + *******************************************************************************/ + public void setProcessName(String processName) + { + this.processName = processName; + } + + + + /******************************************************************************* + ** Fluent setter for processName + *******************************************************************************/ + public RunBackendStepOutput withProcessName(String processName) + { + this.processName = processName; + return (this); + } + + + + /******************************************************************************* + ** Getter for updatedFrontendStepList + *******************************************************************************/ + public List getUpdatedFrontendStepList() + { + return (this.processState.getUpdatedFrontendStepList()); + } + + + + /******************************************************************************* + ** Setter for updatedFrontendStepList + *******************************************************************************/ + public void setUpdatedFrontendStepList(List updatedFrontendStepList) + { + this.processState.setUpdatedFrontendStepList(updatedFrontendStepList); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java index 466e02c4..30a5642a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -327,4 +328,25 @@ public class RunProcessOutput extends AbstractActionOutput implements Serializab { return exception; } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setUpdatedFrontendStepList(List updatedFrontendStepList) + { + this.processState.setUpdatedFrontendStepList(updatedFrontendStepList); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public List getUpdatedFrontendStepList() + { + return this.processState.getUpdatedFrontendStepList(); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ExportInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ExportInput.java index b774ed73..2ed0ba4e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ExportInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ExportInput.java @@ -22,7 +22,6 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting; -import java.io.OutputStream; import java.util.List; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -37,9 +36,8 @@ public class ExportInput extends AbstractTableActionInput private Integer limit; private List fieldNames; - private String filename; - private ReportFormat reportFormat; - private OutputStream reportOutputStream; + private ReportDestination reportDestination; + private String titleRow; private boolean includeHeaderRow = true; @@ -120,71 +118,6 @@ public class ExportInput extends AbstractTableActionInput - /******************************************************************************* - ** Getter for filename - ** - *******************************************************************************/ - public String getFilename() - { - return filename; - } - - - - /******************************************************************************* - ** Setter for filename - ** - *******************************************************************************/ - public void setFilename(String filename) - { - this.filename = filename; - } - - - - /******************************************************************************* - ** Getter for reportFormat - ** - *******************************************************************************/ - public ReportFormat getReportFormat() - { - return reportFormat; - } - - - - /******************************************************************************* - ** Setter for reportFormat - ** - *******************************************************************************/ - public void setReportFormat(ReportFormat reportFormat) - { - this.reportFormat = reportFormat; - } - - - - /******************************************************************************* - ** Getter for reportOutputStream - ** - *******************************************************************************/ - public OutputStream getReportOutputStream() - { - return reportOutputStream; - } - - - - /******************************************************************************* - ** Setter for reportOutputStream - ** - *******************************************************************************/ - public void setReportOutputStream(OutputStream reportOutputStream) - { - this.reportOutputStream = reportOutputStream; - } - - /******************************************************************************* ** @@ -226,4 +159,36 @@ public class ExportInput extends AbstractTableActionInput this.includeHeaderRow = includeHeaderRow; } + + + /******************************************************************************* + ** Getter for reportDestination + *******************************************************************************/ + public ReportDestination getReportDestination() + { + return (this.reportDestination); + } + + + + /******************************************************************************* + ** Setter for reportDestination + *******************************************************************************/ + public void setReportDestination(ReportDestination reportDestination) + { + this.reportDestination = reportDestination; + } + + + + /******************************************************************************* + ** Fluent setter for reportDestination + *******************************************************************************/ + public ExportInput withReportDestination(ReportDestination reportDestination) + { + this.reportDestination = reportDestination; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportDestination.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportDestination.java new file mode 100644 index 00000000..decc8c5f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportDestination.java @@ -0,0 +1,131 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import java.io.OutputStream; + + +/******************************************************************************* + ** Member of report & export Inputs, that wraps details about the destination of + ** where & how the report (or export) is being written. + *******************************************************************************/ +public class ReportDestination +{ + private String filename; + private ReportFormat reportFormat; + private OutputStream reportOutputStream; + + + + /******************************************************************************* + ** Getter for filename + *******************************************************************************/ + public String getFilename() + { + return (this.filename); + } + + + + /******************************************************************************* + ** Setter for filename + *******************************************************************************/ + public void setFilename(String filename) + { + this.filename = filename; + } + + + + /******************************************************************************* + ** Fluent setter for filename + *******************************************************************************/ + public ReportDestination withFilename(String filename) + { + this.filename = filename; + return (this); + } + + + + /******************************************************************************* + ** Getter for reportFormat + *******************************************************************************/ + public ReportFormat getReportFormat() + { + return (this.reportFormat); + } + + + + /******************************************************************************* + ** Setter for reportFormat + *******************************************************************************/ + public void setReportFormat(ReportFormat reportFormat) + { + this.reportFormat = reportFormat; + } + + + + /******************************************************************************* + ** Fluent setter for reportFormat + *******************************************************************************/ + public ReportDestination withReportFormat(ReportFormat reportFormat) + { + this.reportFormat = reportFormat; + return (this); + } + + + + /******************************************************************************* + ** Getter for reportOutputStream + *******************************************************************************/ + public OutputStream getReportOutputStream() + { + return (this.reportOutputStream); + } + + + + /******************************************************************************* + ** Setter for reportOutputStream + *******************************************************************************/ + public void setReportOutputStream(OutputStream reportOutputStream) + { + this.reportOutputStream = reportOutputStream; + } + + + + /******************************************************************************* + ** Fluent setter for reportOutputStream + *******************************************************************************/ + public ReportDestination withReportOutputStream(OutputStream reportOutputStream) + { + this.reportOutputStream = reportOutputStream; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java index c95b1be8..c72056f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java @@ -25,13 +25,13 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting; import java.util.Locale; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.reporting.CsvExportStreamer; -import com.kingsrook.qqq.backend.core.actions.reporting.ExcelExportStreamer; import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; import com.kingsrook.qqq.backend.core.actions.reporting.JsonExportStreamer; import com.kingsrook.qqq.backend.core.actions.reporting.ListOfMapsExportStreamer; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.ExcelPoiBasedStreamingExportStreamer; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.dhatim.fastexcel.Worksheet; +import org.apache.poi.ss.SpreadsheetVersion; /******************************************************************************* @@ -39,15 +39,24 @@ import org.dhatim.fastexcel.Worksheet; *******************************************************************************/ public enum ReportFormat { - XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), - JSON(null, null, JsonExportStreamer::new, "application/json"), - CSV(null, null, CsvExportStreamer::new, "text/csv"), - LIST_OF_MAPS(null, null, ListOfMapsExportStreamer::new, null); + ///////////////////////////////////////////////////////////////////////// + // if we need to fall back to Fastexcel, this was its version of this. // + ///////////////////////////////////////////////////////////////////////// + // XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelFastexcelExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx", true, false, true), + + XLSX(SpreadsheetVersion.EXCEL2007.getMaxRows(), SpreadsheetVersion.EXCEL2007.getMaxColumns(), ExcelPoiBasedStreamingExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx", true, true, true), + JSON(null, null, JsonExportStreamer::new, "application/json", "json", false, false, true), + CSV(null, null, CsvExportStreamer::new, "text/csv", "csv", false, false, false), + LIST_OF_MAPS(null, null, ListOfMapsExportStreamer::new, null, null, false, false, true); private final Integer maxRows; private final Integer maxCols; private final String mimeType; + private final String extension; + private final boolean isBinary; + private final boolean supportsNativePivotTables; + private final boolean supportsMultipleViews; private final Supplier streamerConstructor; @@ -56,12 +65,16 @@ public enum ReportFormat /******************************************************************************* ** *******************************************************************************/ - ReportFormat(Integer maxRows, Integer maxCols, Supplier streamerConstructor, String mimeType) + ReportFormat(Integer maxRows, Integer maxCols, Supplier streamerConstructor, String mimeType, String extension, boolean isBinary, boolean supportsNativePivotTables, boolean supportsMultipleViews) { this.maxRows = maxRows; this.maxCols = maxCols; this.mimeType = mimeType; this.streamerConstructor = streamerConstructor; + this.extension = extension; + this.isBinary = isBinary; + this.supportsNativePivotTables = supportsNativePivotTables; + this.supportsMultipleViews = supportsMultipleViews; } @@ -128,4 +141,48 @@ public enum ReportFormat { return (streamerConstructor.get()); } + + + + /******************************************************************************* + ** Getter for extension + ** + *******************************************************************************/ + public String getExtension() + { + return extension; + } + + + + /******************************************************************************* + ** Getter for isBinary + ** + *******************************************************************************/ + public boolean getIsBinary() + { + return isBinary; + } + + + + /******************************************************************************* + ** Getter for supportsNativePivotTables + ** + *******************************************************************************/ + public boolean getSupportsNativePivotTables() + { + return supportsNativePivotTables; + } + + + + /******************************************************************************* + ** Getter for supportsMultipleViews + ** + *******************************************************************************/ + public boolean getSupportsMultipleViews() + { + return supportsMultipleViews; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormatPossibleValueEnum.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormatPossibleValueEnum.java new file mode 100644 index 00000000..4dec5da9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormatPossibleValueEnum.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; + + +/******************************************************************************* + ** sub-set of ReportFormats to expose as possible-values in-apps + *******************************************************************************/ +public enum ReportFormatPossibleValueEnum implements PossibleValueEnum +{ + XLSX, + CSV, + JSON; + + public static final String NAME = "reportFormat"; + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueId() + { + return name(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return name(); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java index e712a79e..be901e94 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java @@ -22,24 +22,28 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting; -import java.io.OutputStream; import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; /******************************************************************************* - ** Input for an Export action + ** Input for a Report action *******************************************************************************/ public class ReportInput extends AbstractTableActionInput { - private String reportName; + private String reportName; + private QReportMetaData reportMetaData; + private Map inputValues; - private String filename; - private ReportFormat reportFormat; - private OutputStream reportOutputStream; + private ReportDestination reportDestination; + + private Supplier overrideExportStreamerSupplier; @@ -111,66 +115,97 @@ public class ReportInput extends AbstractTableActionInput /******************************************************************************* - ** Getter for filename - ** + ** Getter for reportDestination *******************************************************************************/ - public String getFilename() + public ReportDestination getReportDestination() { - return filename; + return (this.reportDestination); } /******************************************************************************* - ** Setter for filename - ** + ** Setter for reportDestination *******************************************************************************/ - public void setFilename(String filename) + public void setReportDestination(ReportDestination reportDestination) { - this.filename = filename; + this.reportDestination = reportDestination; } /******************************************************************************* - ** Getter for reportFormat - ** + ** Fluent setter for reportDestination *******************************************************************************/ - public ReportFormat getReportFormat() + public ReportInput withReportDestination(ReportDestination reportDestination) { - return reportFormat; + this.reportDestination = reportDestination; + return (this); } /******************************************************************************* - ** Setter for reportFormat - ** + ** Getter for reportMetaData *******************************************************************************/ - public void setReportFormat(ReportFormat reportFormat) + public QReportMetaData getReportMetaData() { - this.reportFormat = reportFormat; + return (this.reportMetaData); } /******************************************************************************* - ** Getter for reportOutputStream - ** + ** Setter for reportMetaData *******************************************************************************/ - public OutputStream getReportOutputStream() + public void setReportMetaData(QReportMetaData reportMetaData) { - return reportOutputStream; + this.reportMetaData = reportMetaData; } /******************************************************************************* - ** Setter for reportOutputStream + ** Fluent setter for reportMetaData + *******************************************************************************/ + public ReportInput withReportMetaData(QReportMetaData reportMetaData) + { + this.reportMetaData = reportMetaData; + return (this); + } + + + + /******************************************************************************* + ** Getter for overrideExportStreamerSupplier ** *******************************************************************************/ - public void setReportOutputStream(OutputStream reportOutputStream) + public Supplier getOverrideExportStreamerSupplier() { - this.reportOutputStream = reportOutputStream; + return overrideExportStreamerSupplier; } + + + + /******************************************************************************* + ** Setter for overrideExportStreamerSupplier + ** + *******************************************************************************/ + public void setOverrideExportStreamerSupplier(Supplier overrideExportStreamerSupplier) + { + this.overrideExportStreamerSupplier = overrideExportStreamerSupplier; + } + + + + /******************************************************************************* + ** Fluent setter for overrideExportStreamerSupplier + ** + *******************************************************************************/ + public ReportInput withOverrideExportStreamerSupplier(Supplier overrideExportStreamerSupplier) + { + this.overrideExportStreamerSupplier = overrideExportStreamerSupplier; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java new file mode 100644 index 00000000..7c51ad8e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; + + +/******************************************************************************* + ** Output for a Report action + *******************************************************************************/ +public class ReportOutput extends AbstractActionOutput implements Serializable +{ + private Integer totalRecordCount; + + + + /******************************************************************************* + ** Getter for totalRecordCount + *******************************************************************************/ + public Integer getTotalRecordCount() + { + return (this.totalRecordCount); + } + + + + /******************************************************************************* + ** Setter for totalRecordCount + *******************************************************************************/ + public void setTotalRecordCount(Integer totalRecordCount) + { + this.totalRecordCount = totalRecordCount; + } + + + + /******************************************************************************* + ** Fluent setter for totalRecordCount + *******************************************************************************/ + public ReportOutput withTotalRecordCount(Integer totalRecordCount) + { + this.totalRecordCount = totalRecordCount; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableDefinition.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableDefinition.java new file mode 100644 index 00000000..991597d3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableDefinition.java @@ -0,0 +1,217 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** Full definition of a pivot table - its rows, columns, and values. + *******************************************************************************/ +public class PivotTableDefinition implements Cloneable, Serializable +{ + private List rows; + private List columns; + private List values; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected PivotTableDefinition clone() throws CloneNotSupportedException + { + PivotTableDefinition clone = (PivotTableDefinition) super.clone(); + + if(rows != null) + { + clone.rows = new ArrayList<>(); + for(PivotTableGroupBy row : rows) + { + clone.rows.add(row.clone()); + } + } + + if(columns != null) + { + clone.columns = new ArrayList<>(); + for(PivotTableGroupBy column : columns) + { + clone.columns.add(column.clone()); + } + } + + if(values != null) + { + clone.values = new ArrayList<>(); + for(PivotTableValue value : values) + { + clone.values.add(value.clone()); + } + } + + return (clone); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PivotTableDefinition withRow(PivotTableGroupBy row) + { + if(this.rows == null) + { + this.rows = new ArrayList<>(); + } + this.rows.add(row); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PivotTableDefinition withColumn(PivotTableGroupBy column) + { + if(this.columns == null) + { + this.columns = new ArrayList<>(); + } + this.columns.add(column); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PivotTableDefinition withValue(PivotTableValue value) + { + if(this.values == null) + { + this.values = new ArrayList<>(); + } + this.values.add(value); + return (this); + } + + + + /******************************************************************************* + ** Getter for rows + *******************************************************************************/ + public List getRows() + { + return (this.rows); + } + + + + /******************************************************************************* + ** Setter for rows + *******************************************************************************/ + public void setRows(List rows) + { + this.rows = rows; + } + + + + /******************************************************************************* + ** Fluent setter for rows + *******************************************************************************/ + public PivotTableDefinition withRows(List rows) + { + this.rows = rows; + return (this); + } + + + + /******************************************************************************* + ** Getter for columns + *******************************************************************************/ + public List getColumns() + { + return (this.columns); + } + + + + /******************************************************************************* + ** Setter for columns + *******************************************************************************/ + public void setColumns(List columns) + { + this.columns = columns; + } + + + + /******************************************************************************* + ** Fluent setter for columns + *******************************************************************************/ + public PivotTableDefinition withColumns(List columns) + { + this.columns = columns; + return (this); + } + + + + /******************************************************************************* + ** Getter for values + *******************************************************************************/ + public List getValues() + { + return (this.values); + } + + + + /******************************************************************************* + ** Setter for values + *******************************************************************************/ + public void setValues(List values) + { + this.values = values; + } + + + + /******************************************************************************* + ** Fluent setter for values + *******************************************************************************/ + public PivotTableDefinition withValues(List values) + { + this.values = values; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableFunction.java new file mode 100644 index 00000000..a3cd1518 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableFunction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +/******************************************************************************* + ** Functions that can be applied to Values in a pivot table. + *******************************************************************************/ +public enum PivotTableFunction +{ + AVERAGE("Average"), + COUNT("Count Values (COUNTA)"), + COUNT_NUMS("Count Numbers (COUNT)"), + MAX("Max"), + MIN("Min"), + PRODUCT("Product"), + STD_DEV("StdDev"), + STD_DEVP("StdDevp"), + SUM("Sum"), + VAR("Var"), + VARP("Varp"); + + + private final String label; + + + + /******************************************************************************* + ** + *******************************************************************************/ + PivotTableFunction(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableGroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableGroupBy.java new file mode 100644 index 00000000..11d1c81e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableGroupBy.java @@ -0,0 +1,143 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Either a row or column grouping in a pivot table. e.g., a field plus + ** sorting details, plus showTotals boolean. + *******************************************************************************/ +public class PivotTableGroupBy implements Cloneable, Serializable +{ + private String fieldName; + private PivotTableOrderBy orderBy; + private boolean showTotals; + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public PivotTableGroupBy withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for orderBy + *******************************************************************************/ + public PivotTableOrderBy getOrderBy() + { + return (this.orderBy); + } + + + + /******************************************************************************* + ** Setter for orderBy + *******************************************************************************/ + public void setOrderBy(PivotTableOrderBy orderBy) + { + this.orderBy = orderBy; + } + + + + /******************************************************************************* + ** Fluent setter for orderBy + *******************************************************************************/ + public PivotTableGroupBy withOrderBy(PivotTableOrderBy orderBy) + { + this.orderBy = orderBy; + return (this); + } + + + + /******************************************************************************* + ** Getter for showTotals + *******************************************************************************/ + public boolean getShowTotals() + { + return (this.showTotals); + } + + + + /******************************************************************************* + ** Setter for showTotals + *******************************************************************************/ + public void setShowTotals(boolean showTotals) + { + this.showTotals = showTotals; + } + + + + /******************************************************************************* + ** Fluent setter for showTotals + *******************************************************************************/ + public PivotTableGroupBy withShowTotals(boolean showTotals) + { + this.showTotals = showTotals; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public PivotTableGroupBy clone() throws CloneNotSupportedException + { + PivotTableGroupBy clone = (PivotTableGroupBy) super.clone(); + return clone; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableOrderBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableOrderBy.java new file mode 100644 index 00000000..d0dadc5e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableOrderBy.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +import java.io.Serializable; + + +/******************************************************************************* + ** How a group-by (rows or columns) should be sorted. + *******************************************************************************/ +public class PivotTableOrderBy implements Serializable +{ + // todo - implement, but only if POI supports (or we build our own support...) +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableValue.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableValue.java new file mode 100644 index 00000000..035b8855 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableValue.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +import java.io.Serializable; + + +/******************************************************************************* + ** a value (e.g., field name + function) used in a pivot table + *******************************************************************************/ +public class PivotTableValue implements Cloneable, Serializable +{ + private String fieldName; + private PivotTableFunction function; + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public PivotTableValue withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for function + *******************************************************************************/ + public PivotTableFunction getFunction() + { + return (this.function); + } + + + + /******************************************************************************* + ** Setter for function + *******************************************************************************/ + public void setFunction(PivotTableFunction function) + { + this.function = function; + } + + + + /******************************************************************************* + ** Fluent setter for function + *******************************************************************************/ + public PivotTableValue withFunction(PivotTableFunction function) + { + this.function = function; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public PivotTableValue clone() throws CloneNotSupportedException + { + PivotTableValue clone = (PivotTableValue) super.clone(); + return clone; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java index 42804602..cc361583 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java @@ -60,6 +60,11 @@ public interface QueryOrGetInputInterface QBackendTransaction getTransaction(); + /******************************************************************************* + ** + *******************************************************************************/ + String getTableName(); + /******************************************************************************* ** Setter for transaction *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java index 750fc91b..87acc4a0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate; import java.io.Serializable; import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -38,6 +39,17 @@ public class GroupBy implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public GroupBy(QFieldMetaData field) + { + this.type = field.getType(); + this.fieldName = field.getName(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java index 3945246e..c7a92518 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.delete; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; @@ -139,6 +140,24 @@ public class DeleteInput extends AbstractTableActionInput + /******************************************************************************* + ** Fluently add 1 primary key to the delete input + ** + *******************************************************************************/ + public DeleteInput withPrimaryKey(Serializable primaryKey) + { + if(primaryKeys == null) + { + primaryKeys = new ArrayList<>(); + } + + primaryKeys.add(primaryKey); + + return (this); + } + + + /******************************************************************************* ** Setter for ids ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java index 4e750818..1753f61f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java @@ -112,7 +112,7 @@ public class InsertInput extends AbstractTableActionInput /******************************************************************************* ** *******************************************************************************/ - public InsertInput withRecordEntities(List recordEntityList) + public InsertInput withRecordEntities(List recordEntityList) { for(QRecordEntity recordEntity : CollectionUtils.nonNullList(recordEntityList)) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java index 8fa6f5ec..26767cce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -37,12 +38,18 @@ import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.security.NullValueBehaviorUtil; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.collections.MutableList; import org.apache.logging.log4j.Level; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -60,11 +67,23 @@ public class JoinsContext private final String mainTableName; private final List queryJoins; + private final QQueryFilter securityFilter; + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // pointer either at securityFilter, or at a sub-filter within it, for when we're doing a recursive build-out of multi-locks // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + private QQueryFilter securityFilterCursor; + //////////////////////////////////////////////////////////////// // note - will have entries for all tables, not just aliases. // //////////////////////////////////////////////////////////////// private final Map aliasToTableNameMap = new HashMap<>(); - private Level logLevel = Level.OFF; + + ///////////////////////////////////////////////////////////////////////////// + // we will get a TON of more output if this gets turned up, so be cautious // + ///////////////////////////////////////////////////////////////////////////// + private Level logLevel = Level.OFF; + private Level logLevelForFilter = Level.OFF; @@ -74,54 +93,225 @@ public class JoinsContext *******************************************************************************/ public JoinsContext(QInstance instance, String tableName, List queryJoins, QQueryFilter filter) throws QException { - log("--- START ----------------------------------------------------------------------", logPair("mainTable", tableName)); this.instance = instance; this.mainTableName = tableName; this.queryJoins = new MutableList<>(queryJoins); + this.securityFilter = new QQueryFilter(); + this.securityFilterCursor = this.securityFilter; + + // log("--- START ----------------------------------------------------------------------", logPair("mainTable", tableName)); + dumpDebug(true, false); for(QueryJoin queryJoin : this.queryJoins) { - log("Processing input query join", logPair("joinTable", queryJoin.getJoinTable()), logPair("alias", queryJoin.getAlias()), logPair("baseTableOrAlias", queryJoin.getBaseTableOrAlias()), logPair("joinMetaDataName", () -> queryJoin.getJoinMetaData().getName())); processQueryJoin(queryJoin); } + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that all tables specified in filter columns are being brought into the query as joins // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + ensureFilterIsRepresented(filter); + logFilter("After ensureFilterIsRepresented:", securityFilter); + + /////////////////////////////////////////////////////////////////////////////////////// + // ensure that any record locks on the main table, which require a join, are present // + /////////////////////////////////////////////////////////////////////////////////////// + MultiRecordSecurityLock multiRecordSecurityLock = RecordSecurityLockFilters.filterForReadLockTree(CollectionUtils.nonNullList(instance.getTable(tableName).getRecordSecurityLocks())); + for(RecordSecurityLock lock : multiRecordSecurityLock.getLocks()) + { + ensureRecordSecurityLockIsRepresented(tableName, tableName, lock, null); + logFilter("After ensureRecordSecurityLockIsRepresented[fieldName=" + lock.getFieldName() + "]:", securityFilter); + } + + /////////////////////////////////////////////////////////////////////////////////// + // make sure that all joins in the query have meta data specified // + // e.g., a user-added join may just specify the join-table // + // or a join implicitly added from a filter may also not have its join meta data // + /////////////////////////////////////////////////////////////////////////////////// + fillInMissingJoinMetaData(); + logFilter("After fillInMissingJoinMetaData:", securityFilter); + /////////////////////////////////////////////////////////////// // ensure any joins that contribute a recordLock are present // /////////////////////////////////////////////////////////////// - for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(instance.getTable(tableName).getRecordSecurityLocks()))) - { - ensureRecordSecurityLockIsRepresented(instance, tableName, recordSecurityLock); - } + ensureAllJoinRecordSecurityLocksAreRepresented(instance); + logFilter("After ensureAllJoinRecordSecurityLocksAreRepresented:", securityFilter); - ensureFilterIsRepresented(filter); - - addJoinsFromExposedJoinPaths(); - - /* todo!! - for(QueryJoin queryJoin : queryJoins) - { - QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); - for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks())) - { - // addCriteriaForRecordSecurityLock(instance, session, joinTable, securityCriteria, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias()); - } - } - */ + //////////////////////////////////////////////////////////////////////////////////// + // if there were any security filters built, then put those into the input filter // + //////////////////////////////////////////////////////////////////////////////////// + addSecurityFiltersToInputFilter(filter); log("Constructed JoinsContext", logPair("mainTableName", this.mainTableName), logPair("queryJoins", this.queryJoins.stream().map(qj -> qj.getJoinTable()).collect(Collectors.joining(",")))); - log("--- END ------------------------------------------------------------------------"); + log("", logPair("securityFilter", securityFilter)); + log("", logPair("fullFilter", filter)); + dumpDebug(false, true); + // log("--- END ------------------------------------------------------------------------"); } /******************************************************************************* - ** + ** Update the input filter with any security filters that were built. *******************************************************************************/ - private void ensureRecordSecurityLockIsRepresented(QInstance instance, String tableName, RecordSecurityLock recordSecurityLock) throws QException + private void addSecurityFiltersToInputFilter(QQueryFilter filter) { + //////////////////////////////////////////////////////////////////////////////////// + // if there's no security filter criteria (including sub-filters), return w/ noop // + //////////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(securityFilter.getSubFilters())) + { + return; + } + + /////////////////////////////////////////////////////////////////////// + // if the input filter is an OR we need to replace it with a new AND // + /////////////////////////////////////////////////////////////////////// + if(filter.getBooleanOperator().equals(QQueryFilter.BooleanOperator.OR)) + { + List originalCriteria = filter.getCriteria(); + List originalSubFilters = filter.getSubFilters(); + + QQueryFilter replacementFilter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); + replacementFilter.setCriteria(originalCriteria); + replacementFilter.setSubFilters(originalSubFilters); + + filter.setCriteria(new ArrayList<>()); + filter.setSubFilters(new ArrayList<>()); + filter.setBooleanOperator(QQueryFilter.BooleanOperator.AND); + filter.addSubFilter(replacementFilter); + } + + filter.addSubFilter(securityFilter); + } + + + + /******************************************************************************* + ** In case we've added any joins to the query that have security locks which + ** weren't previously added to the query, add them now. basically, this is + ** calling ensureRecordSecurityLockIsRepresented for each queryJoin. + *******************************************************************************/ + private void ensureAllJoinRecordSecurityLocksAreRepresented(QInstance instance) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // avoid concurrent modification exceptions by doing a double-loop and breaking the inner any time anything gets added // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Set processedQueryJoins = new HashSet<>(); + boolean addedAnyThisIteration = true; + while(addedAnyThisIteration) + { + addedAnyThisIteration = false; + + for(QueryJoin queryJoin : this.queryJoins) + { + boolean addedAnyForThisJoin = false; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // avoid double-processing the same query join // + // or adding security filters for a join who was only added to the query so that we could add locks (an ImplicitQueryJoinForSecurityLock) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(processedQueryJoins.contains(queryJoin) || queryJoin instanceof ImplicitQueryJoinForSecurityLock) + { + continue; + } + processedQueryJoins.add(queryJoin); + + ////////////////////////////////////////////////////////////////////////////////////////// + // process all locks on this join's join-table. keep track if any new joins were added // + ////////////////////////////////////////////////////////////////////////////////////////// + QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); + + MultiRecordSecurityLock multiRecordSecurityLock = RecordSecurityLockFilters.filterForReadLockTree(CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks())); + for(RecordSecurityLock lock : multiRecordSecurityLock.getLocks()) + { + List addedQueryJoins = ensureRecordSecurityLockIsRepresented(joinTable.getName(), queryJoin.getJoinTableOrItsAlias(), lock, queryJoin); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if any joins were added by this call, add them to the set of processed ones, so they don't get re-processed. // + // also mark the flag that any were added for this join, to manage the double-looping // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(addedQueryJoins)) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make all new joins added in that method be of the same type (inner/left/etc) as the query join they are connected to // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QueryJoin addedQueryJoin : addedQueryJoins) + { + addedQueryJoin.setType(queryJoin.getType()); + } + + processedQueryJoins.addAll(addedQueryJoins); + addedAnyForThisJoin = true; + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////// + // if any new joins were added, we need to break the inner-loop, and continue the outer loop // + // e.g., to process the next query join (but we can't just go back to the foreach queryJoin, // + // because it would fail with concurrent modification) // + /////////////////////////////////////////////////////////////////////////////////////////////// + if(addedAnyForThisJoin) + { + addedAnyThisIteration = true; + break; + } + } + } + } + + + + /******************************************************************************* + ** For a given recordSecurityLock on a given table (with a possible alias), + ** make sure that if any joins are needed to get to the lock, that they are in the query. + ** + ** returns the list of query joins that were added, if any were added + *******************************************************************************/ + private List ensureRecordSecurityLockIsRepresented(String tableName, String tableNameOrAlias, RecordSecurityLock recordSecurityLock, QueryJoin sourceQueryJoin) throws QException + { + List addedQueryJoins = new ArrayList<>(); + + //////////////////////////////////////////////////////////////////////////// + // if this lock is a multi-lock, then recursively process its child-locks // + //////////////////////////////////////////////////////////////////////////// + if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock) + { + log("Processing MultiRecordSecurityLock..."); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // make a new level in the filter-tree - storing old cursor, and updating cursor to point at new level // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter oldSecurityFilterCursor = this.securityFilterCursor; + QQueryFilter nextLevelSecurityFilter = new QQueryFilter(); + this.securityFilterCursor.addSubFilter(nextLevelSecurityFilter); + this.securityFilterCursor = nextLevelSecurityFilter; + + /////////////////////////////////////// + // set the boolean operator to match // + /////////////////////////////////////// + nextLevelSecurityFilter.setBooleanOperator(multiRecordSecurityLock.getOperator().toFilterOperator()); + + ////////////////////// + // process children // + ////////////////////// + for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks())) + { + log(" - Recursive call for childLock: " + childLock); + addedQueryJoins.addAll(ensureRecordSecurityLockIsRepresented(tableName, tableNameOrAlias, childLock, sourceQueryJoin)); + } + + //////////////////// + // restore cursor // + //////////////////// + this.securityFilterCursor = oldSecurityFilterCursor; + + return addedQueryJoins; + } + /////////////////////////////////////////////////////////////////////////////////////////////////// - // ok - so - the join name chain is going to be like this: // - // for a table: orderLineItemExtrinsic (that's 2 away from order, where the security field is): // + // A join name chain is going to look like this: // + // for a table: orderLineItemExtrinsic (that's 2 away from order, where its security field is): // // - securityFieldName = order.clientId // // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic // // so - to navigate from the table to the security field, we need to reverse the joinNameChain, // @@ -129,30 +319,30 @@ public class JoinsContext /////////////////////////////////////////////////////////////////////////////////////////////////// ArrayList joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain())); Collections.reverse(joinNameChain); - log("Evaluating recordSecurityLock", logPair("recordSecurityLock", recordSecurityLock.getFieldName()), logPair("joinNameChain", joinNameChain)); + log("Evaluating recordSecurityLock. Join name chain is of length: " + joinNameChain.size(), logPair("tableNameOrAlias", tableNameOrAlias), logPair("recordSecurityLock", recordSecurityLock.getFieldName()), logPair("joinNameChain", joinNameChain)); - QTableMetaData tmpTable = instance.getTable(mainTableName); + QTableMetaData tmpTable = instance.getTable(tableName); + String securityFieldTableAlias = tableNameOrAlias; + String baseTableOrAlias = tableNameOrAlias; + + boolean chainIsInner = true; + if(sourceQueryJoin != null && QueryJoin.Type.isOuter(sourceQueryJoin.getType())) + { + chainIsInner = false; + } for(String joinName : joinNameChain) { - /////////////////////////////////////////////////////////////////////////////////////////////////////// - // check the joins currently in the query - if any are for this table, then we don't need to add one // - /////////////////////////////////////////////////////////////////////////////////////////////////////// - List matchingJoins = this.queryJoins.stream().filter(queryJoin -> + ////////////////////////////////////////////////////////////////////////////////////////////////// + // check the joins currently in the query - if any are THIS join, then we don't need to add one // + ////////////////////////////////////////////////////////////////////////////////////////////////// + List matchingQueryJoins = this.queryJoins.stream().filter(queryJoin -> { - QJoinMetaData joinMetaData = null; - if(queryJoin.getJoinMetaData() != null) - { - joinMetaData = queryJoin.getJoinMetaData(); - } - else - { - joinMetaData = findJoinMetaData(instance, tableName, queryJoin.getJoinTable()); - } + QJoinMetaData joinMetaData = queryJoin.getJoinMetaData(); return (joinMetaData != null && Objects.equals(joinMetaData.getName(), joinName)); }).toList(); - if(CollectionUtils.nullSafeHasContents(matchingJoins)) + if(CollectionUtils.nullSafeHasContents(matchingQueryJoins)) { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // note - if a user added a join as an outer type, we need to change it to be inner, for the security purpose. // @@ -160,11 +350,40 @@ public class JoinsContext ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// log("- skipping join already in the query", logPair("joinName", joinName)); - if(matchingJoins.get(0).getType().equals(QueryJoin.Type.LEFT) || matchingJoins.get(0).getType().equals(QueryJoin.Type.RIGHT)) + QueryJoin matchedQueryJoin = matchingQueryJoins.get(0); + + if(matchedQueryJoin.getType().equals(QueryJoin.Type.LEFT) || matchedQueryJoin.getType().equals(QueryJoin.Type.RIGHT)) + { + chainIsInner = false; + } + + /* ?? todo ?? + if(matchedQueryJoin.getType().equals(QueryJoin.Type.LEFT) || matchedQueryJoin.getType().equals(QueryJoin.Type.RIGHT)) { log("- - although... it was here as an outer - so switching it to INNER", logPair("joinName", joinName)); - matchingJoins.get(0).setType(QueryJoin.Type.INNER); + matchedQueryJoin.setType(QueryJoin.Type.INNER); } + */ + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // as we're walking from tmpTable to the table which ultimately has the security key field, // + // if the queryJoin we just found is joining out to tmpTable, then we need to advance tmpTable back // + // to the queryJoin's base table - else, tmpTable advances to the matched queryJoin's joinTable // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + if(tmpTable.getName().equals(matchedQueryJoin.getJoinTable())) + { + securityFieldTableAlias = Objects.requireNonNullElse(matchedQueryJoin.getBaseTableOrAlias(), mainTableName); + } + else + { + securityFieldTableAlias = matchedQueryJoin.getJoinTableOrItsAlias(); + } + tmpTable = instance.getTable(securityFieldTableAlias); + + //////////////////////////////////////////////////////////////////////////////////////// + // set the baseTableOrAlias for the next iteration to be this join's joinTableOrAlias // + //////////////////////////////////////////////////////////////////////////////////////// + baseTableOrAlias = securityFieldTableAlias; continue; } @@ -172,20 +391,233 @@ public class JoinsContext QJoinMetaData join = instance.getJoin(joinName); if(join.getLeftTable().equals(tmpTable.getName())) { - QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join).withType(QueryJoin.Type.INNER); - this.addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)"); + securityFieldTableAlias = join.getRightTable() + "_forSecurityJoin_" + join.getName(); + QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock() + .withJoinMetaData(join) + .withType(chainIsInner ? QueryJoin.Type.INNER : QueryJoin.Type.LEFT) + .withBaseTableOrAlias(baseTableOrAlias) + .withAlias(securityFieldTableAlias); + + if(securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.OR) + { + queryJoin.withType(QueryJoin.Type.LEFT); + chainIsInner = false; + } + + addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)", "- "); + addedQueryJoins.add(queryJoin); tmpTable = instance.getTable(join.getRightTable()); } else if(join.getRightTable().equals(tmpTable.getName())) { - QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join.flip()).withType(QueryJoin.Type.INNER); - this.addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)"); + securityFieldTableAlias = join.getLeftTable() + "_forSecurityJoin_" + join.getName(); + QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock() + .withJoinMetaData(join.flip()) + .withType(chainIsInner ? QueryJoin.Type.INNER : QueryJoin.Type.LEFT) + .withBaseTableOrAlias(baseTableOrAlias) + .withAlias(securityFieldTableAlias); + + if(securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.OR) + { + queryJoin.withType(QueryJoin.Type.LEFT); + chainIsInner = false; + } + + addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)", "- "); + addedQueryJoins.add(queryJoin); tmpTable = instance.getTable(join.getLeftTable()); } else { + dumpDebug(false, true); throw (new QException("Error adding security lock joins to query - table name [" + tmpTable.getName() + "] not found in join [" + joinName + "]")); } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for the next iteration of the loop, set the next join's baseTableOrAlias to be the alias we just created // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + baseTableOrAlias = securityFieldTableAlias; + } + + //////////////////////////////////////////////////////////////////////////////////// + // now that we know the joins/tables are in the query, add to the security filter // + //////////////////////////////////////////////////////////////////////////////////// + QueryJoin lastAddedQueryJoin = addedQueryJoins.isEmpty() ? null : addedQueryJoins.get(addedQueryJoins.size() - 1); + if(sourceQueryJoin != null && lastAddedQueryJoin == null) + { + lastAddedQueryJoin = sourceQueryJoin; + } + addSubFilterForRecordSecurityLock(recordSecurityLock, tmpTable, securityFieldTableAlias, !chainIsInner, lastAddedQueryJoin); + + log("Finished evaluating recordSecurityLock"); + + return (addedQueryJoins); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addSubFilterForRecordSecurityLock(RecordSecurityLock recordSecurityLock, QTableMetaData table, String tableNameOrAlias, boolean isOuter, QueryJoin sourceQueryJoin) + { + QSession session = QContext.getQSession(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // check if the key type has an all-access key, and if so, if it's set to true for the current user/session // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); + boolean haveAllAccessKey = false; + if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName())) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we have all-access on this key, then we don't need a criterion for it (as long as we're in an AND filter) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN)) + { + haveAllAccessKey = true; + + if(sourceQueryJoin != null) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // in case the queryJoin object is re-used between queries, and its security criteria need to be different (!!), reset it // + // this can be exposed in tests - maybe not entirely expected in real-world, but seems safe enough // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + sourceQueryJoin.withSecurityCriteria(new ArrayList<>()); + } + + //////////////////////////////////////////////////////////////////////////////////////// + // if we're in an AND filter, then we don't need a criteria for this lock, so return. // + //////////////////////////////////////////////////////////////////////////////////////// + boolean inAnAndFilter = securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.AND; + if(inAnAndFilter) + { + return; + } + } + } + + ///////////////////////////////////////////////////////////////////////////////////////// + // for locks w/o a join chain, the lock fieldName will simply be a field on the table. // + // so just prepend that with the tableNameOrAlias. // + ///////////////////////////////////////////////////////////////////////////////////////// + String fieldName = tableNameOrAlias + "." + recordSecurityLock.getFieldName(); + if(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain())) + { + ///////////////////////////////////////////////////////////////////////////////// + // else, expect a "table.field" in the lock fieldName - but we want to replace // + // the table name part with a possible alias that we took in. // + ///////////////////////////////////////////////////////////////////////////////// + String[] parts = recordSecurityLock.getFieldName().split("\\."); + if(parts.length != 2) + { + dumpDebug(false, true); + throw new IllegalArgumentException("Mal-formatted recordSecurityLock fieldName for lock with joinNameChain in query: " + fieldName); + } + fieldName = tableNameOrAlias + "." + parts[1]; + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // else - get the key values from the session and decide what kind of criterion to build // + /////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter lockFilter = new QQueryFilter(); + List lockCriteria = new ArrayList<>(); + lockFilter.setCriteria(lockCriteria); + + QFieldType type = QFieldType.INTEGER; + try + { + JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = getFieldAndTableNameOrAlias(fieldName); + type = fieldAndTableNameOrAlias.field().getType(); + } + catch(Exception e) + { + LOG.debug("Error getting field type... Trying Integer", e); + } + + if(haveAllAccessKey) + { + //////////////////////////////////////////////////////////////////////////////////////////// + // if we have an all access key (but we got here because we're part of an OR query), then // + // write a criterion that will always be true - e.g., field=field // + //////////////////////////////////////////////////////////////////////////////////////////// + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.TRUE)); + } + else + { + List securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type); + if(CollectionUtils.nullSafeIsEmpty(securityKeyValues)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // handle user with no values -- they can only see null values, and only iff the lock's null-value behavior is ALLOW // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock))) + { + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK)); + } + else + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, if no user/session values, and null-value behavior is deny, then setup a FALSE condition, to allow no rows. // + // todo - maybe avoid running the whole query - as you're not allowed ANY records (based on boolean tree down to this point) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.FALSE)); + } + } + else + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, if user/session has some values, build an IN rule - // + // noting that if the lock's null-value behavior is ALLOW, then we actually want IS_NULL_OR_IN, not just IN // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock))) + { + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_NULL_OR_IN, securityKeyValues)); + } + else + { + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, securityKeyValues)); + } + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there's a sourceQueryJoin, then set the lockCriteria on that join - so it gets written into the JOIN ... ON clause // + // ... unless we're writing an OR filter. then we need the condition in the WHERE clause // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean doNotPutCriteriaInJoinOn = securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.OR; + if(sourceQueryJoin != null && !doNotPutCriteriaInJoinOn) + { + sourceQueryJoin.withSecurityCriteria(lockCriteria); + } + else + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // we used to add an OR IS NULL for cases of an outer-join - but instead, this is now handled by putting the lockCriteria // + // into the join (see above) - so this check is probably deprecated. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /* + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this field is on the outer side of an outer join, then if we do a straight filter on it, then we're basically // + // nullifying the outer join... so for an outer join use-case, OR the security field criteria with a primary-key IS NULL // + // which will make missing rows from the join be found. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(isOuter) + { + if(table == null) + { + table = QContext.getQInstance().getTable(aliasToTableNameMap.get(tableNameOrAlias)); + } + + lockFilter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); + lockFilter.addCriteria(new QFilterCriteria(tableNameOrAlias + "." + table.getPrimaryKeyField(), QCriteriaOperator.IS_BLANK)); + } + */ + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // If this filter isn't for a queryJoin, then just add it to the main list of security sub-filters // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + this.securityFilterCursor.addSubFilter(lockFilter); } } @@ -197,9 +629,9 @@ public class JoinsContext ** use this method to add to the list, instead of ever adding directly, as it's ** important do to that process step (and we've had bugs when it wasn't done). *******************************************************************************/ - private void addQueryJoin(QueryJoin queryJoin, String reason) throws QException + private void addQueryJoin(QueryJoin queryJoin, String reason, String logPrefix) throws QException { - log("Adding query join to context", + log(Objects.requireNonNullElse(logPrefix, "") + "Adding query join to context", logPair("reason", reason), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinMetaData.name", () -> queryJoin.getJoinMetaData().getName()), @@ -208,34 +640,46 @@ public class JoinsContext ); this.queryJoins.add(queryJoin); processQueryJoin(queryJoin); + dumpDebug(false, false); } /******************************************************************************* ** If there are any joins in the context that don't have a join meta data, see - ** if we can find the JoinMetaData to use for them by looking at the main table's - ** exposed joins, and using their join paths. + ** if we can find the JoinMetaData to use for them by looking at all joins in the + ** instance, or at the main table's exposed joins, and using their join paths. *******************************************************************************/ - private void addJoinsFromExposedJoinPaths() throws QException + private void fillInMissingJoinMetaData() throws QException { + log("Begin adding missing join meta data"); + //////////////////////////////////////////////////////////////////////////////// // do a double-loop, to avoid concurrent modification on the queryJoins list. // // that is to say, we'll loop over that list, but possibly add things to it, // // in which case we'll set this flag, and break the inner loop, to go again. // //////////////////////////////////////////////////////////////////////////////// - boolean addedJoin; + Set processedQueryJoins = new HashSet<>(); + boolean addedJoin; do { addedJoin = false; for(QueryJoin queryJoin : queryJoins) { + if(processedQueryJoins.contains(queryJoin)) + { + continue; + } + processedQueryJoins.add(queryJoin); + /////////////////////////////////////////////////////////////////////////////////////////////// // if the join has joinMetaData, then we don't need to process it... unless it needs flipped // /////////////////////////////////////////////////////////////////////////////////////////////// QJoinMetaData joinMetaData = queryJoin.getJoinMetaData(); if(joinMetaData != null) { + log("- QueryJoin already has joinMetaData", logPair("joinMetaDataName", joinMetaData.getName())); + boolean isJoinLeftTableInQuery = false; String joinMetaDataLeftTable = joinMetaData.getLeftTable(); if(joinMetaDataLeftTable.equals(mainTableName)) @@ -265,7 +709,7 @@ public class JoinsContext ///////////////////////////////////////////////////////////////////////////////// if(!isJoinLeftTableInQuery) { - log("Flipping queryJoin because its leftTable wasn't found in the query", logPair("joinMetaDataName", joinMetaData.getName()), logPair("leftTable", joinMetaDataLeftTable)); + log("- - Flipping queryJoin because its leftTable wasn't found in the query", logPair("joinMetaDataName", joinMetaData.getName()), logPair("leftTable", joinMetaDataLeftTable)); queryJoin.setJoinMetaData(joinMetaData.flip()); } } @@ -275,11 +719,13 @@ public class JoinsContext // try to find a direct join between the main table and this table. // // if one is found, then put it (the meta data) on the query join. // ////////////////////////////////////////////////////////////////////// + log("- QueryJoin doesn't have metaData - looking for it", logPair("joinTableOrItsAlias", queryJoin.getJoinTableOrItsAlias())); + String baseTableName = Objects.requireNonNullElse(resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), mainTableName); - QJoinMetaData found = findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable()); + QJoinMetaData found = findJoinMetaData(baseTableName, queryJoin.getJoinTable(), true); if(found != null) { - log("Found joinMetaData - setting it in queryJoin", logPair("joinMetaDataName", found.getName()), logPair("baseTableName", baseTableName), logPair("joinTable", queryJoin.getJoinTable())); + log("- - Found joinMetaData - setting it in queryJoin", logPair("joinMetaDataName", found.getName()), logPair("baseTableName", baseTableName), logPair("joinTable", queryJoin.getJoinTable())); queryJoin.setJoinMetaData(found); } else @@ -293,7 +739,7 @@ public class JoinsContext { if(queryJoin.getJoinTable().equals(exposedJoin.getJoinTable())) { - log("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath())); + log("- - Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath())); ///////////////////////////////////////////////////////////////////////////////////// // loop backward through the join path (from the joinTable back to the main table) // @@ -304,6 +750,7 @@ public class JoinsContext { String joinName = exposedJoin.getJoinPath().get(i); QJoinMetaData joinToAdd = instance.getJoin(joinName); + log("- - - evaluating joinPath element", logPair("i", i), logPair("joinName", joinName)); ///////////////////////////////////////////////////////////////////////////// // get the name from the opposite side of the join (flipping it if needed) // @@ -332,15 +779,22 @@ public class JoinsContext queryJoin.setBaseTableOrAlias(nextTable); } queryJoin.setJoinMetaData(joinToAdd); + log("- - - - this is the last element in the join path, so setting this joinMetaData on the original queryJoin"); } else { QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd); queryJoinToAdd.setType(queryJoin.getType()); addedAnyQueryJoins = true; - this.addQueryJoin(queryJoinToAdd, "forExposedJoin"); + log("- - - - this is not the last element in the join path, so adding a new query join:"); + addQueryJoin(queryJoinToAdd, "forExposedJoin", "- - - - - - "); + dumpDebug(false, false); } } + else + { + log("- - - - join doesn't need added to the query"); + } tmpTable = nextTable; } @@ -361,6 +815,7 @@ public class JoinsContext } while(addedJoin); + log("Done adding missing join meta data"); } @@ -370,12 +825,12 @@ public class JoinsContext *******************************************************************************/ private boolean doesJoinNeedAddedToQuery(String joinName) { - /////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // look at all queryJoins already in context - if any have this join's name, then we don't need this join... // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // look at all queryJoins already in context - if any have this join's name, and aren't implicit-security joins, then we don't need this join... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// for(QueryJoin queryJoin : queryJoins) { - if(queryJoin.getJoinMetaData() != null && queryJoin.getJoinMetaData().getName().equals(joinName)) + if(queryJoin.getJoinMetaData() != null && queryJoin.getJoinMetaData().getName().equals(joinName) && !(queryJoin instanceof ImplicitQueryJoinForSecurityLock)) { return (false); } @@ -395,6 +850,7 @@ public class JoinsContext String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias(); if(aliasToTableNameMap.containsKey(tableNameOrAlias)) { + dumpDebug(false, true); throw (new QException("Duplicate table name or alias: " + tableNameOrAlias)); } aliasToTableNameMap.put(tableNameOrAlias, joinTable.getName()); @@ -439,6 +895,7 @@ public class JoinsContext String[] parts = fieldName.split("\\."); if(parts.length != 2) { + dumpDebug(false, true); throw new IllegalArgumentException("Mal-formatted field name in query: " + fieldName); } @@ -449,6 +906,7 @@ public class JoinsContext QTableMetaData table = instance.getTable(tableName); if(table == null) { + dumpDebug(false, true); throw new IllegalArgumentException("Could not find table [" + tableName + "] in instance for query"); } return new FieldAndTableNameOrAlias(table.getField(baseFieldName), tableOrAlias); @@ -503,17 +961,17 @@ public class JoinsContext for(String filterTable : filterTables) { - log("Evaluating filterTable", logPair("filterTable", filterTable)); + log("Evaluating filter", logPair("filterTable", filterTable)); if(!aliasToTableNameMap.containsKey(filterTable) && !Objects.equals(mainTableName, filterTable)) { - log("- table not in query - adding it", logPair("filterTable", filterTable)); + log("- table not in query - adding a join for it", logPair("filterTable", filterTable)); boolean found = false; for(QJoinMetaData join : CollectionUtils.nonNullMap(QContext.getQInstance().getJoins()).values()) { QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join); if(queryJoin != null) { - this.addQueryJoin(queryJoin, "forFilter (join found in instance)"); + addQueryJoin(queryJoin, "forFilter (join found in instance)", "- - "); found = true; break; } @@ -522,9 +980,13 @@ public class JoinsContext if(!found) { QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER); - this.addQueryJoin(queryJoin, "forFilter (join not found in instance)"); + addQueryJoin(queryJoin, "forFilter (join not found in instance)", "- - "); } } + else + { + log("- table is already in query - not adding any joins", logPair("filterTable", filterTable)); + } } } @@ -566,6 +1028,11 @@ public class JoinsContext getTableNameFromFieldNameAndAddToSet(criteria.getOtherFieldName(), filterTables); } + for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(filter.getOrderBys())) + { + getTableNameFromFieldNameAndAddToSet(orderBy.getFieldName(), filterTables); + } + for(QQueryFilter subFilter : CollectionUtils.nonNullList(filter.getSubFilters())) { populateFilterTablesSet(subFilter, filterTables); @@ -592,7 +1059,7 @@ public class JoinsContext /******************************************************************************* ** *******************************************************************************/ - public QJoinMetaData findJoinMetaData(QInstance instance, String baseTableName, String joinTableName) + public QJoinMetaData findJoinMetaData(String baseTableName, String joinTableName, boolean useExposedJoins) { List matches = new ArrayList<>(); if(baseTableName != null) @@ -644,7 +1111,29 @@ public class JoinsContext } else if(matches.size() > 1) { - throw (new RuntimeException("More than 1 join was found between [" + baseTableName + "] and [" + joinTableName + "]. Specify which one in your QueryJoin.")); + //////////////////////////////////////////////////////////////////////////////// + // if we found more than one join, but we're allowed to useExposedJoins, then // + // see if we can tell which match to used based on the table's exposed joins // + //////////////////////////////////////////////////////////////////////////////// + if(useExposedJoins) + { + QTableMetaData mainTable = QContext.getQInstance().getTable(mainTableName); + for(ExposedJoin exposedJoin : mainTable.getExposedJoins()) + { + if(exposedJoin.getJoinTable().equals(joinTableName)) + { + // todo ... is it wrong to always use 0?? + return instance.getJoin(exposedJoin.getJoinPath().get(0)); + } + } + } + + /////////////////////////////////////////////// + // if we couldn't figure it out, then throw. // + /////////////////////////////////////////////// + dumpDebug(false, true); + throw (new RuntimeException("More than 1 join was found between [" + baseTableName + "] and [" + joinTableName + "] " + + (useExposedJoins ? "(and exposed joins didn't clarify which one to use). " : "") + "Specify which one in your QueryJoin.")); } return (null); @@ -669,4 +1158,79 @@ public class JoinsContext LOG.log(logLevel, message, null, logPairs); } + + /******************************************************************************* + ** + *******************************************************************************/ + private void logFilter(String message, QQueryFilter filter) + { + if(logLevelForFilter.equals(Level.OFF)) + { + return; + } + System.out.println(message + "\n" + filter); + } + + + + /******************************************************************************* + ** Print (to stdout, for easier reading) the object in a big table format for + ** debugging. Happens any time logLevel is > OFF. Not meant for loggly. + *******************************************************************************/ + private void dumpDebug(boolean isStart, boolean isEnd) + { + if(logLevel.equals(Level.OFF)) + { + return; + } + + int sm = 8; + int md = 30; + int lg = 50; + int overhead = 14; + int full = sm + 3 * md + lg + overhead; + + if(isStart) + { + System.out.println("\n" + StringUtils.safeTruncate("--- Start [main table: " + this.mainTableName + "] " + "-".repeat(full), full)); + } + + StringBuilder rs = new StringBuilder(); + String formatString = "| %-" + md + "s | %-" + md + "s %-" + md + "s | %-" + lg + "s | %-" + sm + "s |\n"; + rs.append(String.format(formatString, "Base Table", "Join Table", "(Alias)", "Join Meta Data", "Type")); + String dashesLg = "-".repeat(lg); + String dashesMd = "-".repeat(md); + String dashesSm = "-".repeat(sm); + rs.append(String.format(formatString, dashesMd, dashesMd, dashesMd, dashesLg, dashesSm)); + if(CollectionUtils.nullSafeHasContents(queryJoins)) + { + for(QueryJoin queryJoin : queryJoins) + { + rs.append(String.format( + formatString, + StringUtils.hasContent(queryJoin.getBaseTableOrAlias()) ? StringUtils.safeTruncate(queryJoin.getBaseTableOrAlias(), md) : "--", + StringUtils.safeTruncate(queryJoin.getJoinTable(), md), + (StringUtils.hasContent(queryJoin.getAlias()) ? "(" + StringUtils.safeTruncate(queryJoin.getAlias(), md - 2) + ")" : ""), + queryJoin.getJoinMetaData() == null ? "--" : StringUtils.safeTruncate(queryJoin.getJoinMetaData().getName(), lg), + queryJoin.getType())); + } + } + else + { + rs.append(String.format(formatString, "-empty-", "", "", "", "")); + } + + System.out.print(rs); + + System.out.println(securityFilter); + + if(isEnd) + { + System.out.println(StringUtils.safeTruncate("--- End " + "-".repeat(full), full) + "\n"); + } + else + { + System.out.println(StringUtils.safeTruncate("-".repeat(full), full)); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java index 6dc50b1d..99498007 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java @@ -49,5 +49,7 @@ public enum QCriteriaOperator IS_BLANK, IS_NOT_BLANK, BETWEEN, - NOT_BETWEEN + NOT_BETWEEN, + TRUE, + FALSE } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java index bf14f05f..118aacbf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java @@ -306,6 +306,11 @@ public class QFilterCriteria implements Serializable, Cloneable @Override public String toString() { + if(fieldName == null) + { + return (""); + } + StringBuilder rs = new StringBuilder(fieldName); try { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index 3d07c4c2..dba1a93e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -25,12 +25,18 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.FilterVariableExpression; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -137,7 +143,7 @@ public class QQueryFilter implements Serializable, Cloneable /******************************************************************************* - ** + ** recursively look at both this filter, and any sub-filters it may have. *******************************************************************************/ public boolean hasAnyCriteria() { @@ -150,7 +156,7 @@ public class QQueryFilter implements Serializable, Cloneable { for(QQueryFilter subFilter : subFilters) { - if(subFilter.hasAnyCriteria()) + if(subFilter != null && subFilter.hasAnyCriteria()) { return (true); } @@ -360,23 +366,44 @@ public class QQueryFilter implements Serializable, Cloneable StringBuilder rs = new StringBuilder("("); try { + int criteriaIndex = 0; for(QFilterCriteria criterion : CollectionUtils.nonNullList(criteria)) { - rs.append(criterion).append(" ").append(getBooleanOperator()).append(" "); + if(criteriaIndex > 0) + { + rs.append(" ").append(getBooleanOperator()).append(" "); + } + rs.append(criterion); + criteriaIndex++; } - for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters)) + if(CollectionUtils.nullSafeHasContents(subFilters)) { - rs.append(subFilter); + rs.append("Sub:{"); + int subIndex = 0; + for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters)) + { + if(subIndex > 0) + { + rs.append(" ").append(getBooleanOperator()).append(" "); + } + rs.append(subFilter); + subIndex++; + } + rs.append("}"); } + rs.append(")"); - rs.append("OrderBy["); - for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(orderBys)) + if(CollectionUtils.nullSafeHasContents(orderBys)) { - rs.append(orderBy).append(","); + rs.append("OrderBy["); + for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(orderBys)) + { + rs.append(orderBy).append(","); + } + rs.append("]"); } - rs.append("]"); } catch(Exception e) { @@ -389,6 +416,88 @@ public class QQueryFilter implements Serializable, Cloneable + /******************************************************************************* + ** Replaces any FilterVariables' variableNames with one constructed from the field + ** name, criteria, and index, camel style + ** + *******************************************************************************/ + public void prepForBackend() + { + Map fieldOperatorMap = new HashMap<>(); + for(QFilterCriteria criterion : getCriteria()) + { + if(criterion.getValues() != null) + { + int criteriaIndex = 1; + int valueIndex = 0; + for(Serializable value : criterion.getValues()) + { + /////////////////////////////////////////////////////////////////////////////// + // keep track of what the index is for this criterion, this way if there are // + // more than one with the same id/operator values, we can differentiate // + /////////////////////////////////////////////////////////////////////////////// + String backendName = getBackendName(criterion, valueIndex); + if(!fieldOperatorMap.containsKey(backendName)) + { + fieldOperatorMap.put(backendName, criteriaIndex); + } + else + { + criteriaIndex = fieldOperatorMap.get(backendName) + 1; + fieldOperatorMap.put(backendName, criteriaIndex); + } + + if(value instanceof FilterVariableExpression fve) + { + if(criteriaIndex > 1) + { + backendName += criteriaIndex; + } + fve.setVariableName(backendName); + } + + valueIndex++; + } + } + } + } + + + + /******************************************************************************* + ** builds up a backend name for a field variable expression + ** + *******************************************************************************/ + private String getBackendName(QFilterCriteria criterion, int valueIndex) + { + StringBuilder backendName = new StringBuilder(); + for(String fieldNameParts : criterion.getFieldName().split("\\.")) + { + backendName.append(StringUtils.ucFirst(fieldNameParts)); + } + + for(String operatorParts : criterion.getOperator().name().split("_")) + { + backendName.append(StringUtils.ucFirst(operatorParts.toLowerCase())); + } + + if(criterion.getOperator().equals(QCriteriaOperator.BETWEEN) || criterion.getOperator().equals(QCriteriaOperator.NOT_BETWEEN)) + { + if(valueIndex == 0) + { + backendName.append("From"); + } + else + { + backendName.append("To"); + } + } + + return (StringUtils.lcFirst(backendName.toString())); + } + + + /******************************************************************************* ** Replace any criteria values that look like ${input.XXX} with the value of XXX ** from the supplied inputValues map. @@ -397,8 +506,10 @@ public class QQueryFilter implements Serializable, Cloneable ** QQueryFilter - e.g., if it's one that defined in metaData, and that we don't ** want to be (permanently) changed!! *******************************************************************************/ - public void interpretValues(Map inputValues) + public void interpretValues(Map inputValues) throws QException { + List caughtExceptions = new ArrayList<>(); + QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter(); variableInterpreter.addValueMap("input", inputValues); for(QFilterCriteria criterion : getCriteria()) @@ -409,13 +520,45 @@ public class QQueryFilter implements Serializable, Cloneable for(Serializable value : criterion.getValues()) { - String valueAsString = ValueUtils.getValueAsString(value); - Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString); - newValues.add(interpretedValue); + try + { + if(value instanceof AbstractFilterExpression) + { + /////////////////////////////////////////////////////////////////////// + // if a filter variable expression, evaluate the input values, which // + // will replace the variables with the corresponding actual values // + /////////////////////////////////////////////////////////////////////// + if(value instanceof FilterVariableExpression filterVariableExpression) + { + newValues.add(filterVariableExpression.evaluateInputValues(inputValues)); + } + else + { + newValues.add(value); + } + } + else + { + String valueAsString = ValueUtils.getValueAsString(value); + Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString); + newValues.add(interpretedValue); + } + } + catch(Exception e) + { + caughtExceptions.add(e); + } } criterion.setValues(newValues); } } + + if(!caughtExceptions.isEmpty()) + { + String message = "Error interpreting filter values: " + StringUtils.joinWithCommasAndAnd(caughtExceptions.stream().map(e -> e.getMessage()).toList()); + boolean allUserFacing = caughtExceptions.stream().allMatch(QUserFacingException.class::isInstance); + throw (allUserFacing ? new QUserFacingException(message) : new QException(message)); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java index 0c9c24fb..a0e71e19 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java @@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.util.ArrayList; import java.util.Collection; +import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; @@ -36,7 +38,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterf ** Input data for the Query action ** *******************************************************************************/ -public class QueryInput extends AbstractTableActionInput implements QueryOrGetInputInterface +public class QueryInput extends AbstractTableActionInput implements QueryOrGetInputInterface, Cloneable { private QBackendTransaction transaction; private QQueryFilter filter; @@ -68,6 +70,24 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn private boolean includeAssociations = false; private Collection associationNamesToInclude = null; + private EnumSet queryHints = EnumSet.noneOf(QueryHint.class); + + + + /******************************************************************************* + ** Information about the query that an application (or qqq service) may know and + ** want to tell the backend, that can help influence how the backend processes + ** query. + ** + ** For example, a query with potentially a large result set, for MySQL backend, + ** we may want to configure the result set to stream results rather than do its + ** default in-memory thing. See RDBMSQueryAction for usage. + *******************************************************************************/ + public enum QueryHint + { + POTENTIALLY_LARGE_NUMBER_OF_RESULTS + } + /******************************************************************************* @@ -90,6 +110,40 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInput clone() throws CloneNotSupportedException + { + QueryInput clone = (QueryInput) super.clone(); + + if(fieldsToTranslatePossibleValues != null) + { + clone.fieldsToTranslatePossibleValues = new HashSet<>(fieldsToTranslatePossibleValues); + } + + if(queryJoins != null) + { + clone.queryJoins = new ArrayList<>(queryJoins); + } + + if(clone.associationNamesToInclude != null) + { + clone.associationNamesToInclude = new HashSet<>(associationNamesToInclude); + } + + if(queryHints != null) + { + clone.queryHints = EnumSet.noneOf(QueryHint.class); + clone.queryHints.addAll(queryHints); + } + + return (clone); + } + + + /******************************************************************************* ** Getter for filter ** @@ -569,4 +623,64 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn return (this); } + + + /******************************************************************************* + ** Getter for queryHints + *******************************************************************************/ + public EnumSet getQueryHints() + { + return (this.queryHints); + } + + + + /******************************************************************************* + ** Setter for queryHints + *******************************************************************************/ + public void setQueryHints(EnumSet queryHints) + { + this.queryHints = queryHints; + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withQueryHints(EnumSet queryHints) + { + this.queryHints = queryHints; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withQueryHint(QueryHint queryHint) + { + if(this.queryHints == null) + { + this.queryHints = EnumSet.noneOf(QueryHint.class); + } + this.queryHints.add(queryHint); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for queryHints + *******************************************************************************/ + public QueryInput withoutQueryHint(QueryHint queryHint) + { + if(this.queryHints != null) + { + this.queryHints.remove(queryHint); + } + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java index c1e103e3..b55fbea2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java @@ -22,6 +22,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -49,6 +51,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; ** specific joinMetaData to use must be set. The joinMetaData field can also be ** used instead of specify joinTable and baseTableOrAlias, but only for cases ** where the baseTable is not an alias. + ** + ** The securityCriteria member, in general, is meant to be populated when a + ** JoinsContext is constructed before executing a query, and not meant to be set + ** by users. *******************************************************************************/ public class QueryJoin { @@ -59,13 +65,30 @@ public class QueryJoin private boolean select = false; private Type type = Type.INNER; + private List securityCriteria = new ArrayList<>(); + /******************************************************************************* - ** + ** define the types of joins - INNER, LEFT, RIGHT, or FULL. *******************************************************************************/ public enum Type - {INNER, LEFT, RIGHT, FULL} + { + INNER, + LEFT, + RIGHT, + FULL; + + + + /******************************************************************************* + ** check if a join is an OUTER type (LEFT or RIGHT). + *******************************************************************************/ + public static boolean isOuter(Type type) + { + return (LEFT == type || RIGHT == type); + } + } @@ -348,4 +371,66 @@ public class QueryJoin return (this); } + + + /******************************************************************************* + ** Getter for securityCriteria + *******************************************************************************/ + public List getSecurityCriteria() + { + return (this.securityCriteria); + } + + + + /******************************************************************************* + ** Setter for securityCriteria + *******************************************************************************/ + public void setSecurityCriteria(List securityCriteria) + { + this.securityCriteria = securityCriteria; + } + + + + /******************************************************************************* + ** Fluent setter for securityCriteria + *******************************************************************************/ + public QueryJoin withSecurityCriteria(List securityCriteria) + { + this.securityCriteria = securityCriteria; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for securityCriteria + *******************************************************************************/ + public QueryJoin withSecurityCriteria(QFilterCriteria securityCriteria) + { + if(this.securityCriteria == null) + { + this.securityCriteria = new ArrayList<>(); + } + this.securityCriteria.add(securityCriteria); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "QueryJoin{base=" + + baseTableOrAlias + ", joinTable='" + + joinTable + ", joinMetaData=" + + (joinMetaData == null ? null : joinMetaData.getName()) + ", alias='" + + alias + ", select=" + + select + ", type=" + + type + '}'; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java index da9dad45..bbee71d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java @@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; /******************************************************************************* @@ -89,4 +91,18 @@ public class QueryOutput extends AbstractActionOutput implements Serializable return storage.getRecords(); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public List getRecordEntities(Class entityClass) throws QException + { + List rs = new ArrayList<>(); + for(QRecord record : storage.getRecords()) + { + rs.add(QRecordEntity.fromQRecord(entityClass, record)); + } + return (rs); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java index cc58e9fa..fc4a2034 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions; import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; /******************************************************************************* @@ -33,7 +35,17 @@ public abstract class AbstractFilterExpression implement /******************************************************************************* ** *******************************************************************************/ - public abstract T evaluate(); + public abstract T evaluate() throws QException; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public T evaluateInputValues(Map inputValues) throws QException + { + return (T) this; + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/FilterVariableExpression.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/FilterVariableExpression.java new file mode 100644 index 00000000..5c3be8f7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/FilterVariableExpression.java @@ -0,0 +1,214 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions; + + +import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FilterVariableExpression extends AbstractFilterExpression +{ + private String variableName; + private String fieldName; + private String operator; + private int valueIndex = 0; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Serializable evaluate() throws QException + { + throw (new QUserFacingException("Missing variable value.")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Serializable evaluateInputValues(Map inputValues) throws QException + { + if(!inputValues.containsKey(variableName) || "".equals(ValueUtils.getValueAsString(inputValues.get(variableName)))) + { + throw (new QUserFacingException("Missing value for variable: " + variableName)); + } + return (inputValues.get(variableName)); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public FilterVariableExpression() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + private FilterVariableExpression(String fieldName, int valueIndex) + { + this.fieldName = fieldName; + this.valueIndex = valueIndex; + } + + + + /******************************************************************************* + ** Getter for valueIndex + *******************************************************************************/ + public int getValueIndex() + { + return (this.valueIndex); + } + + + + /******************************************************************************* + ** Setter for valueIndex + *******************************************************************************/ + public void setValueIndex(int valueIndex) + { + this.valueIndex = valueIndex; + } + + + + /******************************************************************************* + ** Fluent setter for valueIndex + *******************************************************************************/ + public FilterVariableExpression withValueIndex(int valueIndex) + { + this.valueIndex = valueIndex; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public FilterVariableExpression withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for variableName + *******************************************************************************/ + public String getVariableName() + { + return (this.variableName); + } + + + + /******************************************************************************* + ** Setter for variableName + *******************************************************************************/ + public void setVariableName(String variableName) + { + this.variableName = variableName; + } + + + + /******************************************************************************* + ** Fluent setter for variableName + *******************************************************************************/ + public FilterVariableExpression withVariableName(String variableName) + { + this.variableName = variableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for operator + *******************************************************************************/ + public String getOperator() + { + return (this.operator); + } + + + + /******************************************************************************* + ** Setter for operator + *******************************************************************************/ + public void setOperator(String operator) + { + this.operator = operator; + } + + + + /******************************************************************************* + ** Fluent setter for operator + *******************************************************************************/ + public FilterVariableExpression withOperator(String operator) + { + this.operator = operator; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/Now.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/Now.java index 9ab54ef7..56edba25 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/Now.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/Now.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions; import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; /******************************************************************************* @@ -35,7 +36,7 @@ public class Now extends AbstractFilterExpression ** *******************************************************************************/ @Override - public Instant evaluate() + public Instant evaluate() throws QException { return (Instant.now()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java index bf36d971..51f64c99 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java @@ -28,6 +28,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.exceptions.QException; /******************************************************************************* @@ -119,7 +120,7 @@ public class NowWithOffset extends AbstractFilterExpression ** *******************************************************************************/ @Override - public Instant evaluate() + public Instant evaluate() throws QException { ///////////////////////////////////////////////////////////////////////////// // Instant doesn't let us plus/minus WEEK, MONTH, or YEAR... // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java index 28e53c37..15fd1bfa 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java @@ -28,6 +28,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.temporal.ChronoField; import java.time.temporal.ChronoUnit; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -95,7 +96,7 @@ public class ThisOrLastPeriod extends AbstractFilterExpression ** *******************************************************************************/ @Override - public Instant evaluate() + public Instant evaluate() throws QException { ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java index 81944a6a..e518dec1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java @@ -39,7 +39,9 @@ public class ReplaceInput extends AbstractTableActionInput private UniqueKey key; private List records; private QQueryFilter filter; - private boolean performDeletes = true; + private boolean performDeletes = true; + private boolean allowNullKeyValuesToEqual = false; + private boolean setPrimaryKeyInInsertedRecords = false; private boolean omitDmlAudit = false; @@ -239,4 +241,66 @@ public class ReplaceInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for allowNullKeyValuesToEqual + *******************************************************************************/ + public boolean getAllowNullKeyValuesToEqual() + { + return (this.allowNullKeyValuesToEqual); + } + + + + /******************************************************************************* + ** Setter for allowNullKeyValuesToEqual + *******************************************************************************/ + public void setAllowNullKeyValuesToEqual(boolean allowNullKeyValuesToEqual) + { + this.allowNullKeyValuesToEqual = allowNullKeyValuesToEqual; + } + + + + /******************************************************************************* + ** Fluent setter for allowNullKeyValuesToEqual + *******************************************************************************/ + public ReplaceInput withAllowNullKeyValuesToEqual(boolean allowNullKeyValuesToEqual) + { + this.allowNullKeyValuesToEqual = allowNullKeyValuesToEqual; + return (this); + } + + + + /******************************************************************************* + ** Getter for setPrimaryKeyInInsertedRecords + *******************************************************************************/ + public boolean getSetPrimaryKeyInInsertedRecords() + { + return (this.setPrimaryKeyInInsertedRecords); + } + + + + /******************************************************************************* + ** Setter for setPrimaryKeyInInsertedRecords + *******************************************************************************/ + public void setSetPrimaryKeyInInsertedRecords(boolean setPrimaryKeyInInsertedRecords) + { + this.setPrimaryKeyInInsertedRecords = setPrimaryKeyInInsertedRecords; + } + + + + /******************************************************************************* + ** Fluent setter for setPrimaryKeyInInsertedRecords + *******************************************************************************/ + public ReplaceInput withSetPrimaryKeyInInsertedRecords(boolean setPrimaryKeyInInsertedRecords) + { + this.setPrimaryKeyInInsertedRecords = setPrimaryKeyInInsertedRecords; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java new file mode 100644 index 00000000..5407faeb --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.storage; + + +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; + + +/******************************************************************************* + ** Input for Storage actions. + *******************************************************************************/ +public class StorageInput extends AbstractTableActionInput +{ + private String reference; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public StorageInput(String storageTableName) + { + super(); + setTableName(storageTableName); + } + + + + /******************************************************************************* + ** Getter for reference + *******************************************************************************/ + public String getReference() + { + return (this.reference); + } + + + + /******************************************************************************* + ** Setter for reference + *******************************************************************************/ + public void setReference(String reference) + { + this.reference = reference; + } + + + + /******************************************************************************* + ** Fluent setter for reference + *******************************************************************************/ + public StorageInput withReference(String reference) + { + this.reference = reference; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java index 7f81a3ae..767b9ee5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java @@ -120,7 +120,7 @@ public class UpdateInput extends AbstractTableActionInput /******************************************************************************* ** *******************************************************************************/ - public UpdateInput withRecordEntities(List recordEntityList) + public UpdateInput withRecordEntities(List recordEntityList) { for(QRecordEntity recordEntity : CollectionUtils.nonNullList(recordEntityList)) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java new file mode 100644 index 00000000..fa06e309 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.common; + + +import java.util.ArrayList; +import java.util.List; +import java.util.TimeZone; +import java.util.function.Function; +import java.util.function.Predicate; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TimeZonePossibleValueSourceMetaDataProvider +{ + public static final String NAME = "timeZones"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QPossibleValueSource produce() + { + return (produce(null, null)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QPossibleValueSource produce(Predicate filter, Function labelMapper) + { + QPossibleValueSource possibleValueSource = new QPossibleValueSource() + .withName("timeZones") + .withType(QPossibleValueSourceType.ENUM) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY); + + List> enumValues = new ArrayList<>(); + for(String availableID : TimeZone.getAvailableIDs()) + { + if(filter == null || filter.test(availableID)) + { + String label = labelMapper == null ? availableID : labelMapper.apply(availableID); + enumValues.add(new QPossibleValue<>(availableID, label)); + } + } + + possibleValueSource.withEnumValues(enumValues); + return (possibleValueSource); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java new file mode 100644 index 00000000..28b60929 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java @@ -0,0 +1,139 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +/******************************************************************************* + ** Model containing datastructure expected by frontend alert widget + ** + *******************************************************************************/ +public class AlertData extends QWidgetData +{ + public enum AlertType + { + ERROR, + SUCCESS, + WARNING + } + + + + private String html; + private AlertType alertType; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AlertData() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AlertData(AlertType alertType, String html) + { + setHtml(html); + setAlertType(alertType); + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public String getType() + { + return WidgetType.ALERT.getType(); + } + + + + /******************************************************************************* + ** Getter for html + ** + *******************************************************************************/ + public String getHtml() + { + return html; + } + + + + /******************************************************************************* + ** Setter for html + ** + *******************************************************************************/ + public void setHtml(String html) + { + this.html = html; + } + + + + /******************************************************************************* + ** Fluent setter for html + ** + *******************************************************************************/ + public AlertData withHtml(String html) + { + this.html = html; + return (this); + } + + + + /******************************************************************************* + ** Getter for alertType + *******************************************************************************/ + public AlertType getAlertType() + { + return (this.alertType); + } + + + + /******************************************************************************* + ** Setter for alertType + *******************************************************************************/ + public void setAlertType(AlertType alertType) + { + this.alertType = alertType; + } + + + + /******************************************************************************* + ** Fluent setter for alertType + *******************************************************************************/ + public AlertData withAlertType(AlertType alertType) + { + this.alertType = alertType; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java index 4d2e79a7..4cfbcb16 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java @@ -54,6 +54,7 @@ public class CompositeWidgetData extends AbstractBlockWidgetData. + */ + +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class DynamicFormWidgetData extends QWidgetData +{ + private List fieldList; + + ///////////////////////////////////////////////////////////////////// + // values for the fields - // + // use a QRecord, so we can do "richer" things, like DisplayValues // + ///////////////////////////////////////////////////////////////////// + private QRecord recordOfFieldValues; + + ///////////////////////////////////////////////////// + // if there are no fields, what message to display // + ///////////////////////////////////////////////////// + private String noFieldsMessage; + + /////////////////////////////////////////////////////////////////////////////////// + // what 1 field do we want to combine the dynamic fields into (as a JSON string) // + /////////////////////////////////////////////////////////////////////////////////// + private String mergedDynamicFormValuesIntoFieldName; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getType() + { + return WidgetType.DYNAMIC_FORM.getType(); + } + + + + /******************************************************************************* + ** Getter for fieldList + *******************************************************************************/ + public List getFieldList() + { + return (this.fieldList); + } + + + + /******************************************************************************* + ** Setter for fieldList + *******************************************************************************/ + public void setFieldList(List fieldList) + { + this.fieldList = fieldList; + } + + + + /******************************************************************************* + ** Fluent setter for fieldList + *******************************************************************************/ + public DynamicFormWidgetData withFieldList(List fieldList) + { + this.fieldList = fieldList; + return (this); + } + + + + /******************************************************************************* + ** Getter for noFieldsMessage + *******************************************************************************/ + public String getNoFieldsMessage() + { + return (this.noFieldsMessage); + } + + + + /******************************************************************************* + ** Setter for noFieldsMessage + *******************************************************************************/ + public void setNoFieldsMessage(String noFieldsMessage) + { + this.noFieldsMessage = noFieldsMessage; + } + + + + /******************************************************************************* + ** Fluent setter for noFieldsMessage + *******************************************************************************/ + public DynamicFormWidgetData withNoFieldsMessage(String noFieldsMessage) + { + this.noFieldsMessage = noFieldsMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for mergedDynamicFormValuesIntoFieldName + *******************************************************************************/ + public String getMergedDynamicFormValuesIntoFieldName() + { + return (this.mergedDynamicFormValuesIntoFieldName); + } + + + + /******************************************************************************* + ** Setter for mergedDynamicFormValuesIntoFieldName + *******************************************************************************/ + public void setMergedDynamicFormValuesIntoFieldName(String mergedDynamicFormValuesIntoFieldName) + { + this.mergedDynamicFormValuesIntoFieldName = mergedDynamicFormValuesIntoFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for mergedDynamicFormValuesIntoFieldName + *******************************************************************************/ + public DynamicFormWidgetData withMergedDynamicFormValuesIntoFieldName(String mergedDynamicFormValuesIntoFieldName) + { + this.mergedDynamicFormValuesIntoFieldName = mergedDynamicFormValuesIntoFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for recordOfFieldValues + *******************************************************************************/ + public QRecord getRecordOfFieldValues() + { + return (this.recordOfFieldValues); + } + + + + /******************************************************************************* + ** Setter for recordOfFieldValues + *******************************************************************************/ + public void setRecordOfFieldValues(QRecord recordOfFieldValues) + { + this.recordOfFieldValues = recordOfFieldValues; + } + + + + /******************************************************************************* + ** Fluent setter for recordOfFieldValues + *******************************************************************************/ + public DynamicFormWidgetData withRecordOfFieldValues(QRecord recordOfFieldValues) + { + this.recordOfFieldValues = recordOfFieldValues; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FieldValueListData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FieldValueListData.java index 33add9cd..6583e7c4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FieldValueListData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FieldValueListData.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -147,7 +148,7 @@ public class FieldValueListData extends QWidgetData } } - QValueFormatter.setDisplayValuesInRecord(fields, record); + QValueFormatter.setDisplayValuesInRecord(null, fields.stream().collect(Collectors.toMap(f -> f.getName(), f -> f)), record); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FilterAndColumnsSetupData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FilterAndColumnsSetupData.java new file mode 100644 index 00000000..10a562a0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FilterAndColumnsSetupData.java @@ -0,0 +1,196 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +import java.util.List; + + +/******************************************************************************* + ** Model containing datastructure expected by frontend filter and columns setup widget + ** + *******************************************************************************/ +public class FilterAndColumnsSetupData extends QWidgetData +{ + private String tableName; + private Boolean allowVariables = false; + private Boolean hideColumns = false; + private List filterDefaultFieldNames; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public FilterAndColumnsSetupData() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public FilterAndColumnsSetupData(String tableName, Boolean allowVariables, Boolean hideColumns, List filterDefaultFieldNames) + { + this.tableName = tableName; + this.allowVariables = allowVariables; + this.hideColumns = hideColumns; + this.filterDefaultFieldNames = filterDefaultFieldNames; + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public String getType() + { + return WidgetType.FILTER_AND_COLUMNS_SETUP.getType(); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public FilterAndColumnsSetupData withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for hideColumns + *******************************************************************************/ + public Boolean getHideColumns() + { + return (this.hideColumns); + } + + + + /******************************************************************************* + ** Setter for hideColumns + *******************************************************************************/ + public void setHideColumns(Boolean hideColumns) + { + this.hideColumns = hideColumns; + } + + + + /******************************************************************************* + ** Fluent setter for hideColumns + *******************************************************************************/ + public FilterAndColumnsSetupData withHideColumns(Boolean hideColumns) + { + this.hideColumns = hideColumns; + return (this); + } + + + + /******************************************************************************* + ** Getter for filterDefaultFieldNames + *******************************************************************************/ + public List getFilterDefaultFieldNames() + { + return (this.filterDefaultFieldNames); + } + + + + /******************************************************************************* + ** Setter for filterDefaultFieldNames + *******************************************************************************/ + public void setFilterDefaultFieldNames(List filterDefaultFieldNames) + { + this.filterDefaultFieldNames = filterDefaultFieldNames; + } + + + + /******************************************************************************* + ** Fluent setter for filterDefaultFieldNames + *******************************************************************************/ + public FilterAndColumnsSetupData withFilterDefaultFieldNames(List filterDefaultFieldNames) + { + this.filterDefaultFieldNames = filterDefaultFieldNames; + return (this); + } + + + + /******************************************************************************* + ** Getter for allowVariables + *******************************************************************************/ + public Boolean getAllowVariables() + { + return (this.allowVariables); + } + + + + /******************************************************************************* + ** Setter for allowVariables + *******************************************************************************/ + public void setAllowVariables(Boolean allowVariables) + { + this.allowVariables = allowVariables; + } + + + + /******************************************************************************* + ** Fluent setter for allowVariables + *******************************************************************************/ + public FilterAndColumnsSetupData withAllowVariables(Boolean allowVariables) + { + this.allowVariables = allowVariables; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/MultiTableData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/MultiTableData.java new file mode 100644 index 00000000..fcd6c068 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/MultiTableData.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +import java.util.List; + + +/******************************************************************************* + ** Model containing datastructure expected by frontend bar chart widget + ** + *******************************************************************************/ +public class MultiTableData extends QWidgetData +{ + List tableDataList; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MultiTableData() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MultiTableData(List tableDataList) + { + setTableDataList(tableDataList); + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public String getType() + { + return WidgetType.MULTI_TABLE.getType(); + } + + + + /******************************************************************************* + ** Getter for tableDataList + *******************************************************************************/ + public List getTableDataList() + { + return (this.tableDataList); + } + + + + /******************************************************************************* + ** Setter for tableDataList + *******************************************************************************/ + public void setTableDataList(List tableDataList) + { + this.tableDataList = tableDataList; + } + + + + /******************************************************************************* + ** Fluent setter for tableDataList + *******************************************************************************/ + public MultiTableData withTableDataList(List tableDataList) + { + this.tableDataList = tableDataList; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java index 421164a5..9989f670 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java @@ -27,9 +27,12 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets; *******************************************************************************/ public enum WidgetType { + /////////////////////////////////// + // (generally) dashboard widgets // + /////////////////////////////////// + ALERT("alert"), BAR_CHART("barChart"), CHART("chart"), - CHILD_RECORD_LIST("childRecordList"), DIVIDER("divider"), FIELD_VALUE_LIST("fieldValueList"), GENERIC("generic"), @@ -39,17 +42,34 @@ public enum WidgetType SMALL_LINE_CHART("smallLineChart"), LOCATION("location"), MULTI_STATISTICS("multiStatistics"), - PARENT_WIDGET("parentWidget"), + MULTI_TABLE("multiTable"), PIE_CHART("pieChart"), - PROCESS("process"), QUICK_SIGHT_CHART("quickSightChart"), STATISTICS("statistics"), STACKED_BAR_CHART("stackedBarChart"), STEPPER("stepper"), TABLE("table"), USA_MAP("usaMap"), + + /////////////////////////////// + // widget to house a process // + /////////////////////////////// + PROCESS("process"), + + /////////////////////// + // container widgets // + /////////////////////// + PARENT_WIDGET("parentWidget"), COMPOSITE("composite"), + + ////////////////////////////// + // record view/edit widgets // + ////////////////////////////// + CHILD_RECORD_LIST("childRecordList"), + DYNAMIC_FORM("dynamicForm"), DATA_BAG_VIEWER("dataBagViewer"), + PIVOT_TABLE_SETUP("pivotTableSetup"), + FILTER_AND_COLUMNS_SETUP("filterAndColumnsSetup"), SCRIPT_VIEWER("scriptViewer"); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java index 8744362c..13ead81c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java @@ -26,6 +26,7 @@ import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; @@ -48,6 +49,11 @@ public @interface QField *******************************************************************************/ String backendName() default ""; + /******************************************************************************* + ** + *******************************************************************************/ + boolean isPrimaryKey() default false; + /******************************************************************************* ** *******************************************************************************/ @@ -88,6 +94,11 @@ public @interface QField *******************************************************************************/ ValueTooLongBehavior valueTooLongBehavior() default ValueTooLongBehavior.PASS_THROUGH; + /******************************************************************************* + ** + *******************************************************************************/ + DynamicDefaultValueBehavior dynamicDefaultValueBehavior() default DynamicDefaultValueBehavior.NONE; + ////////////////////////////////////////////////////////////////////////////////////////// // new attributes here likely need implementation in QFieldMetaData.constructFromGetter // ////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QIgnore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QIgnore.java new file mode 100644 index 00000000..9c702049 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QIgnore.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.data; + + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/******************************************************************************* + ** Marker - that a piece of code should be ignored (e.g., a field not treated as + ** a @QField) + *******************************************************************************/ +@Target({ ElementType.FIELD, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +public @interface QIgnore +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 84a521f3..dffc5947 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -35,12 +35,15 @@ import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.SerializationUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -463,6 +466,17 @@ public class QRecord implements Serializable + /******************************************************************************* + ** Getter for a single field's value + ** + *******************************************************************************/ + public Long getValueLong(String fieldName) + { + return (ValueUtils.getValueAsLong(values.get(fieldName))); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -606,6 +620,22 @@ public class QRecord implements Serializable + /******************************************************************************* + ** Getter for errors + ** + *******************************************************************************/ + @JsonIgnore + public String getErrorsAsString() + { + if(CollectionUtils.nullSafeHasContents(errors)) + { + return StringUtils.join("; ", errors.stream().map(e -> e.getMessage()).toList()); + } + return (""); + } + + + /******************************************************************************* ** Setter for errors ** @@ -722,6 +752,22 @@ public class QRecord implements Serializable + /******************************************************************************* + ** Getter for warnings + ** + *******************************************************************************/ + @JsonIgnore + public String getWarningsAsString() + { + if(CollectionUtils.nullSafeHasContents(warnings)) + { + return StringUtils.join("; ", warnings.stream().map(e -> e.getMessage()).toList()); + } + return (""); + } + + + /******************************************************************************* ** Setter for warnings *******************************************************************************/ @@ -732,6 +778,18 @@ public class QRecord implements Serializable + /******************************************************************************* + ** Fluently Add one warning to this record + ** + *******************************************************************************/ + public QRecord withWarning(QWarningMessage warning) + { + addWarning(warning); + return (this); + } + + + /******************************************************************************* ** Fluent setter for warnings *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java index 622ad1b5..89de8bea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java @@ -218,6 +218,7 @@ public abstract class QRecordEntity } + /******************************************************************************* ** *******************************************************************************/ @@ -296,7 +297,19 @@ public abstract class QRecordEntity } else { - LOG.debug("Skipping field without @QField annotation", logPair("class", c.getSimpleName()), logPair("fieldName", fieldName)); + Optional ignoreAnnotation = getQIgnoreAnnotation(c, fieldName); + Optional associationAnnotation = getQAssociationAnnotation(c, fieldName); + + if(ignoreAnnotation.isPresent() || associationAnnotation.isPresent()) + { + //////////////////////////////////////////////////////////// + // silently skip if marked as an association or an ignore // + //////////////////////////////////////////////////////////// + } + else + { + LOG.debug("Skipping field without @QField annotation", logPair("class", c.getSimpleName()), logPair("fieldName", fieldName)); + } } } else @@ -360,6 +373,16 @@ public abstract class QRecordEntity + /******************************************************************************* + ** + *******************************************************************************/ + public static Optional getQIgnoreAnnotation(Class c, String ignoreName) + { + return (getAnnotationOnField(c, QIgnore.class, ignoreName)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -419,9 +442,9 @@ public abstract class QRecordEntity } else { - if(!method.getName().equals("getClass")) + if(!method.getName().equals("getClass") && method.getAnnotation(QIgnore.class) == null) { - LOG.debug("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported."); + LOG.debug("Method [" + method.getName() + "] in [" + method.getDeclaringClass().getSimpleName() + "] looks like a getter, but its return type, [" + method.getReturnType().getSimpleName() + "], isn't supported."); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java index f5530b3b..f2350825 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java @@ -145,7 +145,7 @@ public interface QRecordEnum { if(!method.getName().equals("getClass") && !method.getName().equals("getDeclaringClass") && !method.getName().equals("getPossibleValueId")) { - LOG.debug("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported."); + LOG.debug("Method [" + method.getName() + "] in [" + method.getDeclaringClass().getSimpleName() + "] looks like a getter, but its return type, [" + method.getReturnType().getSimpleName() + "], isn't supported."); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java index 4207a132..2756715f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java @@ -30,7 +30,7 @@ import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; ** MetaDataProducerHelper, to put point at a package full of these, and populate ** your whole QInstance. *******************************************************************************/ -public abstract class MetaDataProducer implements MetaDataProducerInterface +public abstract class MetaDataProducer implements MetaDataProducerInterface { } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index b6ea0975..54c7ae4b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -27,11 +27,16 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import com.google.common.collect.ImmutableSet; import com.google.common.reflect.ClassPath; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -43,7 +48,30 @@ public class MetaDataProducerHelper { private static final QLogger LOG = QLogger.getLogger(MetaDataProducerHelper.class); + private static Map, Integer> comparatorValuesByType = new HashMap<>(); + private static Integer defaultComparatorValue; + private static ImmutableSet topLevelClasses; + + static + { + //////////////////////////////////////////////////////////////////////////////////////// + // define how we break ties in sort-order based on the meta-dta type. e.g., do apps // + // after all other types (as apps often try to get other types from the instance) // + // also - do backends earlier than others (e.g., tables may expect backends to exist) // + // any types not in the map get the default value. // + //////////////////////////////////////////////////////////////////////////////////////// + comparatorValuesByType.put(QBackendMetaData.class, 1); + + ///////////////////////////////////// + // unspecified ones will come here // + ///////////////////////////////////// + defaultComparatorValue = 10; + + comparatorValuesByType.put(QJoinMetaData.class, 21); + comparatorValuesByType.put(QWidgetMetaData.class, 22); + comparatorValuesByType.put(QAppMetaData.class, 23); + } /******************************************************************************* ** Recursively find all classes in the given package, that implement MetaDataProducerInterface @@ -102,11 +130,9 @@ public class MetaDataProducerHelper } } - //////////////////////////////////////////////////////////////////////////////////////////// - // sort them by sort order, then by the type that they return - specifically - doing apps // - // after all other types (as apps often try to get other types from the instance) // - // also - do backends earlier than others (e.g., tables may expect backends to exist) // - //////////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////////// + // sort them by sort order, then by the type that they return, as set up in the static map // + ///////////////////////////////////////////////////////////////////////////////////////////// producers.sort(Comparator .comparing((MetaDataProducerInterface p) -> p.getSortOrder()) .thenComparing((MetaDataProducerInterface p) -> @@ -114,18 +140,7 @@ public class MetaDataProducerHelper try { Class outputType = p.getClass().getMethod("produce", QInstance.class).getReturnType(); - if(outputType.equals(QAppMetaData.class)) - { - return (2); - } - else if(outputType.equals(QBackendMetaData.class)) - { - return (0); - } - else - { - return (1); - } + return comparatorValuesByType.getOrDefault(outputType, defaultComparatorValue); } catch(Exception e) { @@ -142,7 +157,7 @@ public class MetaDataProducerHelper { try { - TopLevelMetaDataInterface metaData = producer.produce(instance); + MetaDataProducerOutput metaData = producer.produce(instance); if(metaData != null) { metaData.addSelfToInstance(instance); @@ -150,7 +165,7 @@ public class MetaDataProducerHelper } catch(Exception e) { - LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e); + LOG.warn("error executing metaDataProducer", e, logPair("producer", producer.getClass().getSimpleName())); } } else @@ -172,7 +187,7 @@ public class MetaDataProducerHelper List> classes = new ArrayList<>(); ClassLoader loader = Thread.currentThread().getContextClassLoader(); - for(ClassPath.ClassInfo info : ClassPath.from(loader).getTopLevelClasses()) + for(ClassPath.ClassInfo info : getTopLevelClasses(loader)) { if(info.getName().startsWith(packageName)) { @@ -183,4 +198,29 @@ public class MetaDataProducerHelper return (classes); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private static ImmutableSet getTopLevelClasses(ClassLoader loader) throws IOException + { + if(topLevelClasses == null) + { + topLevelClasses = ClassPath.from(loader).getTopLevelClasses(); + } + + return (topLevelClasses); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void clearTopLevelClassCache() + { + topLevelClasses = null; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java new file mode 100644 index 00000000..d6797a2a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java @@ -0,0 +1,101 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Output object for a MetaDataProducer, which contains multiple meta-data + ** objects. + *******************************************************************************/ +public class MetaDataProducerMultiOutput implements MetaDataProducerOutput +{ + private List contents; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addSelfToInstance(QInstance instance) + { + for(MetaDataProducerOutput metaDataProducerOutput : CollectionUtils.nonNullList(contents)) + { + metaDataProducerOutput.addSelfToInstance(instance); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void add(MetaDataProducerOutput metaDataProducerOutput) + { + if(contents == null) + { + contents = new ArrayList<>(); + } + contents.add(metaDataProducerOutput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MetaDataProducerMultiOutput with(MetaDataProducerOutput metaDataProducerOutput) + { + add(metaDataProducerOutput); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public List getEach(Class c) + { + List rs = new ArrayList<>(); + + for(MetaDataProducerOutput content : contents) + { + if(content instanceof MetaDataProducerMultiOutput multiOutput) + { + rs.addAll(multiOutput.getEach(c)); + } + else if(c.isInstance(content)) + { + rs.add(c.cast(content)); + } + } + + return (rs); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerOutput.java new file mode 100644 index 00000000..74504e72 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerOutput.java @@ -0,0 +1,40 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +/******************************************************************************* + ** Interface to mark objects that can be produced by a MetaDataProducer. + ** + ** These would usually be TopLevelMetaData objects (a table, a process, etc) + ** but can also be a MetaDataProducerMultiOutput, to produce multiple objects + ** from one producer. + *******************************************************************************/ +public interface MetaDataProducerOutput +{ + + /******************************************************************************* + ** call the appropriate methods on a QInstance to add ourselves to it. + *******************************************************************************/ + void addSelfToInstance(QInstance instance); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index eb63b7f4..628fa870 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -155,18 +155,6 @@ public class QBackendMetaData implements TopLevelMetaDataInterface - /******************************************************************************* - ** Fluent setter, returning generically, to help sub-class fluent flows - *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withBackendType(String backendType) - { - this.backendType = backendType; - return (T) this; - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 115fcccc..1c5ecbe6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode; import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNodeType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; @@ -53,8 +54,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import io.github.cdimascio.dotenv.Dotenv; @@ -76,6 +79,7 @@ public class QInstance private QAuthenticationMetaData authentication = null; private QBrandingMetaData branding = null; private Map automationProviders = new HashMap<>(); + private Map messagingProviders = new HashMap<>(); //////////////////////////////////////////////////////////////////////////////////////////// // Important to use LinkedHashmap here, to preserve the order in which entries are added. // @@ -91,6 +95,9 @@ public class QInstance private Map queueProviders = new LinkedHashMap<>(); private Map queues = new LinkedHashMap<>(); + private Map schedulers = new LinkedHashMap<>(); + private Map schedulableTypes = new LinkedHashMap<>(); + private Map supplementalMetaData = new LinkedHashMap<>(); private String deploymentMode; @@ -734,6 +741,53 @@ public class QInstance + /******************************************************************************* + ** + *******************************************************************************/ + public void addMessagingProvider(QMessagingProviderMetaData messagingProvider) + { + String name = messagingProvider.getName(); + if(this.messagingProviders.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second messagingProvider with name: " + name)); + } + this.messagingProviders.put(name, messagingProvider); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QMessagingProviderMetaData getMessagingProvider(String name) + { + return (this.messagingProviders.get(name)); + } + + + + /******************************************************************************* + ** Getter for messagingProviders + ** + *******************************************************************************/ + public Map getMessagingProviders() + { + return messagingProviders; + } + + + + /******************************************************************************* + ** Setter for messagingProviders + ** + *******************************************************************************/ + public void setMessagingProviders(Map messagingProviders) + { + this.messagingProviders = messagingProviders; + } + + + /******************************************************************************* ** Getter for hasBeenValidated ** @@ -1224,4 +1278,106 @@ public class QInstance metaData.addSelfToInstance(this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addScheduler(QSchedulerMetaData scheduler) + { + String name = scheduler.getName(); + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a scheduler without a name.")); + } + if(this.schedulers.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second scheduler with name: " + name)); + } + this.schedulers.put(name, scheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QSchedulerMetaData getScheduler(String name) + { + return (this.schedulers.get(name)); + } + + + + /******************************************************************************* + ** Getter for schedulers + ** + *******************************************************************************/ + public Map getSchedulers() + { + return schedulers; + } + + + + /******************************************************************************* + ** Setter for schedulers + ** + *******************************************************************************/ + public void setSchedulers(Map schedulers) + { + this.schedulers = schedulers; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addSchedulableType(SchedulableType schedulableType) + { + String name = schedulableType.getName(); + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a schedulableType without a name.")); + } + if(this.schedulableTypes.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second schedulableType with name: " + name)); + } + this.schedulableTypes.put(name, schedulableType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public SchedulableType getSchedulableType(String name) + { + return (this.schedulableTypes.get(name)); + } + + + + /******************************************************************************* + ** Getter for schedulableTypes + ** + *******************************************************************************/ + public Map getSchedulableTypes() + { + return schedulableTypes; + } + + + + /******************************************************************************* + ** Setter for schedulableTypes + ** + *******************************************************************************/ + public void setSchedulableTypes(Map schedulableTypes) + { + this.schedulableTypes = schedulableTypes; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java index 7be8db54..116bae9e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java @@ -26,9 +26,14 @@ package com.kingsrook.qqq.backend.core.model.metadata; ** Interface for meta-data classes that can be added directly (e.g, at the top ** level) to a QInstance (such as a QTableMetaData - not a QFieldMetaData). *******************************************************************************/ -public interface TopLevelMetaDataInterface +public interface TopLevelMetaDataInterface extends MetaDataProducerOutput { + /******************************************************************************* + ** + *******************************************************************************/ + String getName(); + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java index 6dca9cde..31af5cff 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java @@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.automation; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; /******************************************************************************* @@ -35,8 +34,6 @@ public class QAutomationProviderMetaData implements TopLevelMetaDataInterface private String name; private QAutomationProviderType type; - private QScheduleMetaData schedule; - /******************************************************************************* @@ -107,40 +104,6 @@ public class QAutomationProviderMetaData implements TopLevelMetaDataInterface - /******************************************************************************* - ** Getter for schedule - ** - *******************************************************************************/ - public QScheduleMetaData getSchedule() - { - return schedule; - } - - - - /******************************************************************************* - ** Setter for schedule - ** - *******************************************************************************/ - public void setSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - } - - - - /******************************************************************************* - ** Fluent setter for schedule - ** - *******************************************************************************/ - public QAutomationProviderMetaData withSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - return (this); - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java index 6e2d7541..6eb01b58 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java @@ -55,6 +55,17 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getName() + { + return "Branding"; + } + + + /******************************************************************************* ** Getter for companyName ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java index 03688222..e7793b03 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -24,11 +24,15 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard; import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; @@ -61,7 +65,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface protected Map icons; - protected Map helpContent; + protected Map> helpContent; protected Map defaultValues = new LinkedHashMap<>(); @@ -691,10 +695,11 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface } + /******************************************************************************* ** Getter for helpContent *******************************************************************************/ - public Map getHelpContent() + public Map> getHelpContent() { return (this.helpContent); } @@ -704,7 +709,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface /******************************************************************************* ** Setter for helpContent *******************************************************************************/ - public void setHelpContent(Map helpContent) + public void setHelpContent(Map> helpContent) { this.helpContent = helpContent; } @@ -714,11 +719,49 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface /******************************************************************************* ** Fluent setter for helpContent *******************************************************************************/ - public QWidgetMetaData withHelpContent(Map helpContent) + public QWidgetMetaData withHelpContent(Map> helpContent) { this.helpContent = helpContent; return (this); } + + /******************************************************************************* + ** Fluent setter for adding 1 helpContent (for a slot) + *******************************************************************************/ + public QWidgetMetaData withHelpContent(String slot, QHelpContent helpContent) + { + if(this.helpContent == null) + { + this.helpContent = new HashMap<>(); + } + + List listForSlot = this.helpContent.computeIfAbsent(slot, (k) -> new ArrayList<>()); + QInstanceHelpContentManager.putHelpContentInList(helpContent, listForSlot); + + return (this); + } + + + + /******************************************************************************* + ** remove a helpContent for a slot based on its set of roles + *******************************************************************************/ + public void removeHelpContent(String slot, Set roles) + { + if(this.helpContent == null) + { + return; + } + + List listForSlot = this.helpContent.get(slot); + if(listForSlot == null) + { + return; + } + + QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java index 1c702e39..ed8af4e3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java @@ -25,10 +25,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard; import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.Set; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; @@ -235,7 +237,7 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, T /******************************************************************************* ** *******************************************************************************/ - default Map getHelpContent() + default Map> getHelpContent() { return (null); } @@ -244,11 +246,29 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, T /******************************************************************************* ** *******************************************************************************/ - default void setHelpContent(Map helpContent) + default void setHelpContent(Map> helpContent) { LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)"); } + /******************************************************************************* + ** + *******************************************************************************/ + default QWidgetMetaDataInterface withHelpContent(String slot, QHelpContent helpContent) + { + LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)"); + return (this); + } + + /******************************************************************************* + ** remove a helpContent for a slot based on its set of roles + *******************************************************************************/ + default void removeHelpContent(String slot, Set roles) + { + LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)"); + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java index d5446dea..37bd9d23 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java @@ -28,6 +28,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard; *******************************************************************************/ public class WidgetDropdownData { + private String name; private String possibleValueSourceName; private String foreignKeyFieldName; private String label; @@ -44,6 +45,9 @@ public class WidgetDropdownData //////////////////////////////////////////////////////////////////////////////////////////////// private String labelForNullValue; + private WidgetDropdownType type = WidgetDropdownType.POSSIBLE_VALUE_SOURCE; + + /******************************************************************************* ** Getter for possibleValueSourceName @@ -366,4 +370,65 @@ public class WidgetDropdownData } + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public WidgetDropdownType getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(WidgetDropdownType type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public WidgetDropdownData withType(WidgetDropdownType type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public WidgetDropdownData withName(String name) + { + this.name = name; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/PlainExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownType.java similarity index 82% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/PlainExcelStyler.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownType.java index 3fdd189a..be47959f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/PlainExcelStyler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownType.java @@ -19,13 +19,15 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting; +package com.kingsrook.qqq.backend.core.model.metadata.dashboard; /******************************************************************************* - ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface. + ** Possible types for widget dropdowns + ** *******************************************************************************/ -public class PlainExcelStyler implements ExcelStylerInterface +public enum WidgetDropdownType { - + POSSIBLE_VALUE_SOURCE, + DATE_PICKER } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java new file mode 100644 index 00000000..64104583 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java @@ -0,0 +1,359 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.fields; + + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Field Display Behavior class for customizing the display values used + ** in date-time fields + *******************************************************************************/ +public class DateTimeDisplayValueBehavior implements FieldDisplayBehavior +{ + private static final QLogger LOG = QLogger.getLogger(DateTimeDisplayValueBehavior.class); + + private String zoneIdFromFieldName; + private String fallbackZoneId; + + private String defaultZoneId; + + private static DateTimeDisplayValueBehavior NOOP = new DateTimeDisplayValueBehavior(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DateTimeDisplayValueBehavior getDefault() + { + return NOOP; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + if(StringUtils.hasContent(defaultZoneId)) + { + applyDefaultZoneId(recordList, table, field); + } + else if(StringUtils.hasContent(zoneIdFromFieldName)) + { + applyZoneIdFromFieldName(recordList, table, field); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void applyDefaultZoneId(List recordList, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + try + { + Instant instant = record.getValueInstant(field.getName()); + + if(instant == null) + { + continue; + } + + ZonedDateTime zonedDateTime = instant.atZone(ZoneId.of(defaultZoneId)); + record.setDisplayValue(field.getName(), QValueFormatter.formatDateTimeWithZone(zonedDateTime)); + } + catch(Exception e) + { + LOG.info("Error applying defaultZoneId DateTimeDisplayValueBehavior", logPair("table", table.getName()), logPair("field", field.getName()), logPair("id", record.getValue(table.getPrimaryKeyField()))); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void applyZoneIdFromFieldName(List recordList, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + try + { + Instant instant = record.getValueInstant(field.getName()); + if(instant == null) + { + continue; + } + + String zoneString = record.getValueString(zoneIdFromFieldName); + + ZoneId zoneId = null; + if(StringUtils.hasContent(zoneString)) + { + try + { + zoneId = ZoneId.of(zoneString); + } + catch(Exception e) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // we probably(?) don't need a stack trace here (and it could get noisy?), so just info w/ the exception message... // + // and we expect this might be somewhat frequent, if you might have invalid values in your zoneId field... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.info("Exception applying zoneIdFromFieldName behavior", logPair("message", e.getMessage()), logPair("table", table.getName()), logPair("field", field.getName()), logPair("id", record.getValue(table.getPrimaryKeyField()))); + } + } + + if(zoneId == null) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // if the zone string from the other field isn't valid, and we have a fallback, try to use it // + //////////////////////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(fallbackZoneId)) + { + //////////////////////////////////////////////////////////////////////////////////////////// + // assume that validation has confirmed this is a valid zone - so no try-catch right here // + //////////////////////////////////////////////////////////////////////////////////////////// + zoneId = ZoneId.of(fallbackZoneId); + } + } + + if(zoneId != null) + { + ZonedDateTime zonedDateTime = instant.atZone(zoneId); + record.setDisplayValue(field.getName(), QValueFormatter.formatDateTimeWithZone(zonedDateTime)); + } + } + catch(Exception e) + { + /////////////////////////////////////////////////////////////////////// + // we don't expect this to ever hit - so warn it w/ stack if it does // + /////////////////////////////////////////////////////////////////////// + LOG.warn("Unexpected error applying zoneIdFromFieldName behavior", e, logPair("table", table.getName()), logPair("field", field.getName()), logPair("id", record.getValue(table.getPrimaryKeyField()))); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData) + { + List errors = new ArrayList<>(); + String errorSuffix = " field [" + fieldMetaData.getName() + "] in table [" + tableMetaData.getName() + "]"; + + if(!QFieldType.DATE_TIME.equals(fieldMetaData.getType())) + { + errors.add("A DateTimeDisplayValueBehavior was a applied to a non-DATE_TIME" + errorSuffix); + } + + ////////////////////////////////////////////////// + // validate rules if zoneIdFromFieldName is set // + ////////////////////////////////////////////////// + if(StringUtils.hasContent(zoneIdFromFieldName)) + { + if(StringUtils.hasContent(defaultZoneId)) + { + errors.add("You may not specify both zoneIdFromFieldName and defaultZoneId in DateTimeDisplayValueBehavior on" + errorSuffix); + } + + if(!tableMetaData.getFields().containsKey(zoneIdFromFieldName)) + { + errors.add("Unrecognized field name [" + zoneIdFromFieldName + "] for [zoneIdFromFieldName] in DateTimeDisplayValueBehavior on" + errorSuffix); + } + else + { + QFieldMetaData zoneIdField = tableMetaData.getFields().get(zoneIdFromFieldName); + if(!QFieldType.STRING.equals(zoneIdField.getType())) + { + errors.add("A non-STRING type [" + zoneIdField.getType() + "] was specified as the zoneIdFromFieldName field [" + zoneIdFromFieldName + "] in DateTimeDisplayValueBehavior on" + errorSuffix); + } + } + } + + //////////////////////////////////////////// + // validate rules if defaultZoneId is set // + //////////////////////////////////////////// + if(StringUtils.hasContent(defaultZoneId)) + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // would check that you didn't specify from zoneIdFromFieldName - but that's covered above // + ///////////////////////////////////////////////////////////////////////////////////////////// + + if(StringUtils.hasContent(fallbackZoneId)) + { + errors.add("You may not specify both defaultZoneId and fallbackZoneId in DateTimeDisplayValueBehavior on" + errorSuffix); + } + + try + { + ZoneId.of(defaultZoneId); + } + catch(Exception e) + { + errors.add("Invalid ZoneId [" + defaultZoneId + "] for [defaultZoneId] in DateTimeDisplayValueBehavior on" + errorSuffix + "; " + e.getMessage()); + } + } + + ///////////////////////////////////////////// + // validate rules if fallbackZoneId is set // + ///////////////////////////////////////////// + if(StringUtils.hasContent(fallbackZoneId)) + { + if(!StringUtils.hasContent(zoneIdFromFieldName)) + { + errors.add("You may only set fallbackZoneId if using zoneIdFromFieldName in DateTimeDisplayValueBehavior on" + errorSuffix); + } + + try + { + ZoneId.of(fallbackZoneId); + } + catch(Exception e) + { + errors.add("Invalid ZoneId [" + fallbackZoneId + "] for [fallbackZoneId] in DateTimeDisplayValueBehavior on" + errorSuffix + "; " + e.getMessage()); + } + } + + return (errors); + } + + + + /******************************************************************************* + ** Getter for zoneIdFromFieldName + *******************************************************************************/ + public String getZoneIdFromFieldName() + { + return (this.zoneIdFromFieldName); + } + + + + /******************************************************************************* + ** Setter for zoneIdFromFieldName + *******************************************************************************/ + public void setZoneIdFromFieldName(String zoneIdFromFieldName) + { + this.zoneIdFromFieldName = zoneIdFromFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for zoneIdFromFieldName + *******************************************************************************/ + public DateTimeDisplayValueBehavior withZoneIdFromFieldName(String zoneIdFromFieldName) + { + this.zoneIdFromFieldName = zoneIdFromFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for defaultZoneId + *******************************************************************************/ + public String getDefaultZoneId() + { + return (this.defaultZoneId); + } + + + + /******************************************************************************* + ** Setter for defaultZoneId + *******************************************************************************/ + public void setDefaultZoneId(String defaultZoneId) + { + this.defaultZoneId = defaultZoneId; + } + + + + /******************************************************************************* + ** Fluent setter for defaultZoneId + *******************************************************************************/ + public DateTimeDisplayValueBehavior withDefaultZoneId(String defaultZoneId) + { + this.defaultZoneId = defaultZoneId; + return (this); + } + + + + /******************************************************************************* + ** Getter for fallbackZoneId + *******************************************************************************/ + public String getFallbackZoneId() + { + return (this.fallbackZoneId); + } + + + + /******************************************************************************* + ** Setter for fallbackZoneId + *******************************************************************************/ + public void setFallbackZoneId(String fallbackZoneId) + { + this.fallbackZoneId = fallbackZoneId; + } + + + + /******************************************************************************* + ** Fluent setter for fallbackZoneId + *******************************************************************************/ + public DateTimeDisplayValueBehavior withFallbackZoneId(String fallbackZoneId) + { + this.fallbackZoneId = fallbackZoneId; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java index eed23a56..91ae5d4f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java @@ -49,7 +49,6 @@ public interface DisplayFormat /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:Indentation") static String getExcelFormat(String javaDisplayFormat) { if(javaDisplayFormat == null) @@ -58,21 +57,21 @@ public interface DisplayFormat } return switch(javaDisplayFormat) - { - case DisplayFormat.DEFAULT -> null; - case DisplayFormat.COMMAS -> "#,##0"; - case DisplayFormat.DECIMAL1 -> "0.0"; - case DisplayFormat.DECIMAL2 -> "0.00"; - case DisplayFormat.DECIMAL3 -> "0.000"; - case DisplayFormat.DECIMAL1_COMMAS -> "#,##0.0"; - case DisplayFormat.DECIMAL2_COMMAS -> "#,##0.00"; - case DisplayFormat.DECIMAL3_COMMAS -> "#,##0.000"; - case DisplayFormat.CURRENCY -> "$#,##0.00"; - case DisplayFormat.PERCENT -> "0%"; - case DisplayFormat.PERCENT_POINT1 -> "0.0%"; - case DisplayFormat.PERCENT_POINT2 -> "0.00%"; - default -> null; - }; + { + case DisplayFormat.DEFAULT -> null; + case DisplayFormat.COMMAS -> "#,##0"; + case DisplayFormat.DECIMAL1 -> "0.0"; + case DisplayFormat.DECIMAL2 -> "0.00"; + case DisplayFormat.DECIMAL3 -> "0.000"; + case DisplayFormat.DECIMAL1_COMMAS -> "#,##0.0"; + case DisplayFormat.DECIMAL2_COMMAS -> "#,##0.00"; + case DisplayFormat.DECIMAL3_COMMAS -> "#,##0.000"; + case DisplayFormat.CURRENCY -> "$#,##0.00"; + case DisplayFormat.PERCENT -> "0%"; + case DisplayFormat.PERCENT_POINT1 -> "0.0%"; + case DisplayFormat.PERCENT_POINT2 -> "0.00%"; + default -> null; + }; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java index 0a782774..dbf47e6d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -26,13 +26,15 @@ import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.util.List; -import java.util.Set; import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -45,6 +47,7 @@ public enum DynamicDefaultValueBehavior implements FieldBehavior recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set> behaviorsToOmit) + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) { if(this.equals(NONE)) { return; } - if(behaviorsToOmit != null && behaviorsToOmit.contains(this)) - { - return; - } switch(this) { case CREATE_DATE -> applyCreateDate(action, recordList, table, field); case MODIFY_DATE -> applyModifyDate(action, recordList, table, field); + case USER_ID -> applyUserId(action, recordList, table, field); default -> throw new IllegalStateException("Unexpected enum value: " + this); } } @@ -136,6 +136,27 @@ public enum DynamicDefaultValueBehavior implements FieldBehavior recordList, QTableMetaData table, QFieldMetaData field) + { + String fieldName = field.getName(); + String userId = ObjectUtils.tryElse(() -> QContext.getQSession().getUser().getIdReference(), null); + if(StringUtils.hasContent(userId)) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + if(!StringUtils.hasContent(record.getValueString(fieldName))) + { + record.setValue(field.getName(), userId); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java index 215843df..6b2d92bf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -22,8 +22,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; +import java.util.Collections; import java.util.List; -import java.util.Set; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -34,8 +35,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; ** Interface for (expected to be?) enums which define behaviors that get applied ** to fields. ** - ** At the present, these behaviors get applied before a field is stored (insert - ** or update), through the ValueBehaviorApplier class. + ** Some of these behaviors get applied before a field is stored (insert + ** or update), through the ValueBehaviorApplier class. Others can be used to + ** do more advanced display formatting than the displayFormat string alone can + ** do (see QValueFormatter). ** *******************************************************************************/ public interface FieldBehavior> @@ -45,12 +48,13 @@ public interface FieldBehavior> ** In case a behavior of this type wasn't set on the field, what should the ** default of this type be? *******************************************************************************/ + @JsonIgnore T getDefault(); /******************************************************************************* ** Apply this behavior to a list of records *******************************************************************************/ - void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set> behaviorsToOmit); + void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field); /******************************************************************************* ** control if multiple behaviors of this type should be allowed together on a field. @@ -60,4 +64,14 @@ public interface FieldBehavior> return (false); } + /******************************************************************************* + ** allow this behavior to be validated during QInstance validation. + ** + ** return a list of validation errors, if there are any. + *******************************************************************************/ + default List validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData) + { + return (Collections.emptyList()); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java new file mode 100644 index 00000000..c5150557 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java @@ -0,0 +1,31 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.fields; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface FieldDisplayBehavior> extends FieldBehavior +{ + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index 3a72dedb..d78d469d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -221,11 +221,16 @@ public class QFieldMetaData implements Cloneable setMaxLength(fieldAnnotation.maxLength()); } - if(fieldAnnotation.valueTooLongBehavior() != ValueTooLongBehavior.PASS_THROUGH) + if(fieldAnnotation.valueTooLongBehavior() != ValueTooLongBehavior.values()[0].getDefault()) { withBehavior(fieldAnnotation.valueTooLongBehavior()); } + if(fieldAnnotation.dynamicDefaultValueBehavior() != DynamicDefaultValueBehavior.values()[0].getDefault()) + { + withBehavior(fieldAnnotation.dynamicDefaultValueBehavior()); + } + if(StringUtils.hasContent(fieldAnnotation.defaultValue())) { ValueUtils.getValueAsFieldType(this.type, fieldAnnotation.defaultValue()); @@ -716,6 +721,17 @@ public class QFieldMetaData implements Cloneable { return (behaviorType.getEnumConstants()[0].getDefault()); } + else + { + try + { + return (behaviorType.getConstructor().newInstance().getDefault()); + } + catch(Exception e) + { + LOG.warn("Error getting default behaviorType for [" + behaviorType.getSimpleName() + "]", e); + } + } return (null); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index 8971a846..b0e8f8e7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -37,6 +37,7 @@ public enum QFieldType { STRING, INTEGER, + LONG, DECIMAL, BOOLEAN, DATE, @@ -65,6 +66,10 @@ public enum QFieldType { return (INTEGER); } + if(c.equals(Long.class) || c.equals(long.class)) + { + return (LONG); + } if(c.equals(BigDecimal.class)) { return (DECIMAL); @@ -110,7 +115,7 @@ public enum QFieldType *******************************************************************************/ public boolean isNumeric() { - return this == QFieldType.INTEGER || this == QFieldType.DECIMAL; + return this == QFieldType.INTEGER || this == QFieldType.LONG || this == QFieldType.DECIMAL; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java index 77cd24cd..f439f7e6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; import java.util.List; -import java.util.Set; import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -66,16 +65,12 @@ public enum ValueTooLongBehavior implements FieldBehavior ** *******************************************************************************/ @Override - public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set> behaviorsToOmit) + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) { if(this.equals(PASS_THROUGH)) { return; } - if(behaviorsToOmit != null && behaviorsToOmit.contains(this)) - { - return; - } String fieldName = field.getName(); if(!QFieldType.STRING.equals(field.getType())) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index 4df7b4af..ef93e024 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; @@ -75,6 +76,8 @@ public class QFrontendTableMetaData private boolean usesVariants; private String variantTableLabel; + private ShareableTableMetaData shareableTableMetaData; + ////////////////////////////////////////////////////////////////////////////////// // do not add setters. take values from the source-object in the constructor!! // ////////////////////////////////////////////////////////////////////////////////// @@ -104,6 +107,8 @@ public class QFrontendTableMetaData } this.sections = tableMetaData.getSections(); + + this.shareableTableMetaData = tableMetaData.getShareableTableMetaData(); } if(includeJoins) @@ -367,4 +372,14 @@ public class QFrontendTableMetaData return (this.variantTableLabel); } + + + /******************************************************************************* + ** Getter for shareableTableMetaData + ** + *******************************************************************************/ + public ShareableTableMetaData getShareableTableMetaData() + { + return shareableTableMetaData; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java index 374650bf..4312949f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend; +import java.io.Serializable; import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonInclude; @@ -58,8 +59,9 @@ public class QFrontendWidgetMetaData private boolean showReloadButton = false; private boolean showExportButton = false; - protected Map icons; - protected Map helpContent; + protected Map icons; + protected Map> helpContent; + protected Map defaultValues; private final boolean hasPermission; @@ -95,6 +97,7 @@ public class QFrontendWidgetMetaData } this.helpContent = widgetMetaData.getHelpContent(); + this.defaultValues = widgetMetaData.getDefaultValues(); hasPermission = PermissionsHelper.hasWidgetPermission(actionInput, name); } @@ -270,8 +273,20 @@ public class QFrontendWidgetMetaData ** Getter for helpContent ** *******************************************************************************/ - public Map getHelpContent() + public Map> getHelpContent() { return helpContent; } + + + + /******************************************************************************* + ** Getter for defaultValues + ** + *******************************************************************************/ + public Map getDefaultValues() + { + return defaultValues; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinType.java index dfbf812b..c3e8db08 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinType.java @@ -44,14 +44,13 @@ public enum JoinType /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") public JoinType flip() { return switch(this) - { - case ONE_TO_MANY -> MANY_TO_ONE; - case MANY_TO_ONE -> ONE_TO_MANY; - case MANY_TO_MANY, ONE_TO_ONE -> this; - }; + { + case ONE_TO_MANY -> MANY_TO_ONE; + case MANY_TO_ONE -> ONE_TO_MANY; + case MANY_TO_MANY, ONE_TO_ONE -> this; + }; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java index 946fa448..4db07304 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.layout; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -363,11 +365,11 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu /******************************************************************************* ** *******************************************************************************/ - public QAppMetaData withSectionOfChildren(QAppSection section, QAppChildMetaData... children) + public QAppMetaData withSectionOfChildren(QAppSection section, Collection children) { this.addSection(section); - for(QAppChildMetaData child : children) + for(QAppChildMetaData child : CollectionUtils.nonNullCollection(children)) { withChild(child); if(child instanceof QTableMetaData) @@ -392,6 +394,15 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu } + /******************************************************************************* + ** + *******************************************************************************/ + public QAppMetaData withSectionOfChildren(QAppSection section, QAppChildMetaData... children) + { + return (withSectionOfChildren(section, children == null ? null : Arrays.stream(children).toList())); + } + + /******************************************************************************* ** Getter for permissionRules diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/QMessagingProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/QMessagingProviderMetaData.java new file mode 100644 index 00000000..116104ea --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/QMessagingProviderMetaData.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; + + +/******************************************************************************* + ** Base class for qqq messaging-providers. e.g., a connection to an outbound + ** email service, or, for example, Slack. + *******************************************************************************/ +public class QMessagingProviderMetaData implements TopLevelMetaDataInterface +{ + private String name; + private String type; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public QMessagingProviderMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public String getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public QMessagingProviderMetaData withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addMessagingProvider(this); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProvider.java new file mode 100644 index 00000000..df920919 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProvider.java @@ -0,0 +1,56 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging.email; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput; +import com.kingsrook.qqq.backend.core.modules.messaging.MessagingProviderInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class EmailMessagingProvider implements MessagingProviderInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getType() + { + return (EmailMessagingProviderMetaData.TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SendMessageOutput sendMessage(SendMessageInput sendMessageInput) throws QException + { + return new SendEmailAction().sendMessage(sendMessageInput); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProviderMetaData.java new file mode 100644 index 00000000..7fdb931a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProviderMetaData.java @@ -0,0 +1,116 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging.email; + + +import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData; +import com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class EmailMessagingProviderMetaData extends QMessagingProviderMetaData +{ + private String smtpServer; + private String smtpPort; + + public static final String TYPE = "EMAIL"; + + static + { + QMessagingProviderDispatcher.registerMessagingProvider(new EmailMessagingProvider()); + } + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public EmailMessagingProviderMetaData() + { + super(); + setType(TYPE); + } + + + + /******************************************************************************* + ** Getter for smtpServer + *******************************************************************************/ + public String getSmtpServer() + { + return (this.smtpServer); + } + + + + /******************************************************************************* + ** Setter for smtpServer + *******************************************************************************/ + public void setSmtpServer(String smtpServer) + { + this.smtpServer = smtpServer; + } + + + + /******************************************************************************* + ** Fluent setter for smtpServer + *******************************************************************************/ + public EmailMessagingProviderMetaData withSmtpServer(String smtpServer) + { + this.smtpServer = smtpServer; + return (this); + } + + + + /******************************************************************************* + ** Getter for smtpPort + *******************************************************************************/ + public String getSmtpPort() + { + return (this.smtpPort); + } + + + + /******************************************************************************* + ** Setter for smtpPort + *******************************************************************************/ + public void setSmtpPort(String smtpPort) + { + this.smtpPort = smtpPort; + } + + + + /******************************************************************************* + ** Fluent setter for smtpPort + *******************************************************************************/ + public EmailMessagingProviderMetaData withSmtpPort(String smtpPort) + { + this.smtpPort = smtpPort; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/SendEmailAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/SendEmailAction.java new file mode 100644 index 00000000..2723be6f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/SendEmailAction.java @@ -0,0 +1,216 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging.email; + + +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Content; +import com.kingsrook.qqq.backend.core.model.actions.messaging.MultiParty; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Party; +import com.kingsrook.qqq.backend.core.model.actions.messaging.PartyRole; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailContentRole; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailPartyRole; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import jakarta.mail.Address; +import jakarta.mail.Message; +import jakarta.mail.Multipart; +import jakarta.mail.Session; +import jakarta.mail.Transport; +import jakarta.mail.internet.AddressException; +import jakarta.mail.internet.InternetAddress; +import jakarta.mail.internet.MimeBodyPart; +import jakarta.mail.internet.MimeMessage; +import jakarta.mail.internet.MimeMultipart; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SendEmailAction +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public SendMessageOutput sendMessage(SendMessageInput sendMessageInput) throws QException + { + EmailMessagingProviderMetaData messagingProvider = (EmailMessagingProviderMetaData) QContext.getQInstance().getMessagingProvider(sendMessageInput.getMessagingProviderName()); + + ///////////////////////////////////////// + // set up properties to make a session // + ///////////////////////////////////////// + Properties properties = new Properties(); + properties.setProperty("mail.smtp.host", messagingProvider.getSmtpServer()); + properties.setProperty("mail.smtp.port", messagingProvider.getSmtpPort()); + Session session = Session.getInstance(properties); + + try + { + //////////////////////////////////////////// + // Construct a default MimeMessage object // + //////////////////////////////////////////// + MimeMessage emailMessage = new MimeMessage(session); + emailMessage.setSubject(sendMessageInput.getSubject()); + + Party to = sendMessageInput.getTo(); + if(to instanceof MultiParty toMultiParty) + { + for(Party party : toMultiParty.getPartyList()) + { + addRecipient(emailMessage, party); + } + } + else + { + addRecipient(emailMessage, to); + } + + Party from = sendMessageInput.getFrom(); + if(from instanceof MultiParty fromMultiParty) + { + for(Party party : fromMultiParty.getPartyList()) + { + addSender(emailMessage, party); + } + } + else + { + addSender(emailMessage, from); + } + + Multipart multipart = new MimeMultipart(); + for(Content content : sendMessageInput.getContentList()) + { + if(EmailContentRole.HTML.equals(content.getContentRole())) + { + MimeBodyPart mimeBodyPart = new MimeBodyPart(); + mimeBodyPart.setContent(content.getBody(), "text/html; charset=utf-8"); + multipart.addBodyPart(mimeBodyPart); + } + else if(EmailContentRole.TEXT.equals(content.getContentRole())) + { + MimeBodyPart mimeBodyPart = new MimeBodyPart(); + mimeBodyPart.setContent(content.getBody(), "text/plain; charset=utf-8"); + multipart.addBodyPart(mimeBodyPart); + } + } + + emailMessage.setContent(multipart); + + ///////////// + // send it // + ///////////// + Transport.send(emailMessage); + System.out.println("Message dispatched successfully..."); + } + catch(Exception e) + { + throw (new QException("Error sending email", e)); + } + + return null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addSender(MimeMessage emailMessage, Party party) throws Exception + { + if(EmailPartyRole.REPLY_TO.equals(party.getRole())) + { + InternetAddress internetAddress = getInternetAddressFromParty(party); + Address[] replyTo = emailMessage.getReplyTo(); + if(replyTo == null || replyTo.length == 0) + { + emailMessage.setReplyTo(new Address[] { internetAddress }); + } + else + { + List
replyToList = Arrays.asList(replyTo); + emailMessage.setReplyTo(replyToList.toArray(new Address[0])); + } + } + else if(party.getRole() == null || PartyRole.Default.DEFAULT.equals(party.getRole()) || EmailPartyRole.FROM.equals(party.getRole())) + { + emailMessage.setFrom(getInternetAddressFromParty(party)); + } + else + { + throw (new QException("Unrecognized sender role [" + party.getRole() + "]")); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addRecipient(MimeMessage emailMessage, Party party) throws Exception + { + Message.RecipientType recipientType; + if(EmailPartyRole.CC.equals(party.getRole())) + { + recipientType = Message.RecipientType.CC; + } + else if(EmailPartyRole.BCC.equals(party.getRole())) + { + recipientType = Message.RecipientType.BCC; + } + else if(party.getRole() == null || PartyRole.Default.DEFAULT.equals(party.getRole()) || EmailPartyRole.TO.equals(party.getRole())) + { + recipientType = Message.RecipientType.TO; + } + else + { + throw (new QException("Unrecognized recipient role [" + party.getRole() + "]")); + } + + InternetAddress internetAddress = getInternetAddressFromParty(party); + emailMessage.addRecipient(recipientType, internetAddress); + System.out.println("add recipient: [" + recipientType + "] => [" + internetAddress + "]"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static InternetAddress getInternetAddressFromParty(Party party) throws AddressException, UnsupportedEncodingException + { + InternetAddress internetAddress = new InternetAddress(party.getAddress()); + if(StringUtils.hasContent(party.getLabel())) + { + internetAddress.setPersonal(party.getLabel()); + } + return internetAddress; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SESMessagingProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SESMessagingProvider.java new file mode 100644 index 00000000..8f08974f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SESMessagingProvider.java @@ -0,0 +1,56 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging.ses; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput; +import com.kingsrook.qqq.backend.core.modules.messaging.MessagingProviderInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SESMessagingProvider implements MessagingProviderInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getType() + { + return (SESMessagingProviderMetaData.TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SendMessageOutput sendMessage(SendMessageInput sendMessageInput) throws QException + { + return new SendSESAction().sendMessage(sendMessageInput); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SESMessagingProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SESMessagingProviderMetaData.java new file mode 100644 index 00000000..24a18514 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SESMessagingProviderMetaData.java @@ -0,0 +1,148 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging.ses; + + +import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData; +import com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SESMessagingProviderMetaData extends QMessagingProviderMetaData +{ + private String accessKey; + private String secretKey; + private String region; + + public static final String TYPE = "SES"; + + static + { + QMessagingProviderDispatcher.registerMessagingProvider(new SESMessagingProvider()); + } + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SESMessagingProviderMetaData() + { + super(); + setType(TYPE); + } + + + + /******************************************************************************* + ** Getter for accessKey + *******************************************************************************/ + public String getAccessKey() + { + return (this.accessKey); + } + + + + /******************************************************************************* + ** Setter for accessKey + *******************************************************************************/ + public void setAccessKey(String accessKey) + { + this.accessKey = accessKey; + } + + + + /******************************************************************************* + ** Fluent setter for accessKey + *******************************************************************************/ + public SESMessagingProviderMetaData withAccessKey(String accessKey) + { + this.accessKey = accessKey; + return (this); + } + + + + /******************************************************************************* + ** Getter for secretKey + *******************************************************************************/ + public String getSecretKey() + { + return (this.secretKey); + } + + + + /******************************************************************************* + ** Setter for secretKey + *******************************************************************************/ + public void setSecretKey(String secretKey) + { + this.secretKey = secretKey; + } + + + + /******************************************************************************* + ** Fluent setter for secretKey + *******************************************************************************/ + public SESMessagingProviderMetaData withSecretKey(String secretKey) + { + this.secretKey = secretKey; + return (this); + } + + + + /******************************************************************************* + ** Getter for region + *******************************************************************************/ + public String getRegion() + { + return (this.region); + } + + + + /******************************************************************************* + ** Setter for region + *******************************************************************************/ + public void setRegion(String region) + { + this.region = region; + } + + + + /******************************************************************************* + ** Fluent setter for region + *******************************************************************************/ + public SESMessagingProviderMetaData withRegion(String region) + { + this.region = region; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java new file mode 100644 index 00000000..53e13f54 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java @@ -0,0 +1,335 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging.ses; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailService; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClientBuilder; +import com.amazonaws.services.simpleemail.model.Body; +import com.amazonaws.services.simpleemail.model.Content; +import com.amazonaws.services.simpleemail.model.Destination; +import com.amazonaws.services.simpleemail.model.Message; +import com.amazonaws.services.simpleemail.model.SendEmailRequest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.messaging.MultiParty; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Party; +import com.kingsrook.qqq.backend.core.model.actions.messaging.PartyRole; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailContentRole; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailPartyRole; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SendSESAction +{ + private static final QLogger LOG = QLogger.getLogger(SendSESAction.class); + + private AmazonSimpleEmailService amazonSES; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public SendMessageOutput sendMessage(SendMessageInput sendMessageInput) throws QException + { + try + { + AmazonSimpleEmailService client = getAmazonSES(sendMessageInput); + + /////////////////////////////////// + // build up a send email request // + /////////////////////////////////// + SendEmailRequest request = new SendEmailRequest() + .withSource(getSource(sendMessageInput)) + .withReplyToAddresses(getReplyTos(sendMessageInput)) + .withDestination(buildDestination(sendMessageInput)) + .withMessage(buildMessage(sendMessageInput)); + + client.sendEmail(request); + LOG.info("SES Message [" + request.getMessage().getSubject().getData() + "] was sent to [" + request.getDestination().toString() + "]."); + } + catch(Exception e) + { + String message = "An unexpected error occurred sending an SES message."; + throw (new QException(message, e)); + } + + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + Message buildMessage(SendMessageInput input) throws QException + { + /////////////////////////////////////////////////////////////////////////////////////////////// + // iterate over all contents of our input, looking for an HTML and Text version of the email // + /////////////////////////////////////////////////////////////////////////////////////////////// + Body body = new Body(); + for(com.kingsrook.qqq.backend.core.model.actions.messaging.Content content : CollectionUtils.nonNullList(input.getContentList())) + { + if(EmailContentRole.TEXT.equals(content.getContentRole())) + { + body.setText(new Content().withCharset("UTF-8").withData(content.getBody())); + } + else if(EmailContentRole.HTML.equals(content.getContentRole())) + { + body.setHtml(new Content().withCharset("UTF-8").withData(content.getBody())); + } + } + + //////////////////////////////////////////////// + // error if no text or html body was provided // + //////////////////////////////////////////////// + if(body.getText() == null && body.getHtml() == null) + { + throw (new QException("Cannot send SES message because neither a 'Text' nor an 'HTML' body was provided.")); + } + + //////////////////////////////////////// + // warning if no subject was provided // + //////////////////////////////////////// + Message message = new Message(); + message.setBody(body); + + ///////////////////////////////////// + // warn if no subject was provided // + ///////////////////////////////////// + if(input.getSubject() == null) + { + LOG.warn("Sending SES message with no subject."); + } + else + { + message.setSubject(new Content().withCharset("UTF-8").withData(input.getSubject())); + } + + return (message); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + List getReplyTos(SendMessageInput input) throws QException + { + //////////////////////////// + // no input, no reply tos // + //////////////////////////// + if(input == null) + { + return (Collections.emptyList()); + } + + /////////////////////////////////////// + // build up a list of froms if multi // + /////////////////////////////////////// + List partyList = getPartyListFromParty(input.getFrom()); + if(partyList == null) + { + return (Collections.emptyList()); + } + + /////////////////////////////// + // only get reply to parties // + /////////////////////////////// + List replyToParties = partyList.stream().filter(p -> EmailPartyRole.REPLY_TO.equals(p.getRole())).toList(); + + ////////////////////////////////// + // get addresses from reply tos // + ////////////////////////////////// + List replyTos = replyToParties.stream().map(Party::getAddress).toList(); + + ///////////////////////////// + // return the from address // + ///////////////////////////// + return (replyTos); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + String getSource(SendMessageInput input) throws QException + { + /////////////////////////////// + // error if no from provided // + /////////////////////////////// + if(input.getFrom() == null) + { + throw (new QException("Cannot send SES message because a FROM was not provided.")); + } + + /////////////////////////////////////// + // build up a list of froms if multi // + /////////////////////////////////////// + List partyList = getPartyListFromParty(input.getFrom()); + + /////////////////////////////////////// + // remove any roles that aren't FROM // + /////////////////////////////////////// + partyList.removeIf(p -> p.getRole() != null && !EmailPartyRole.FROM.equals(p.getRole())); + + /////////////////////////////////////////////////////////////////////////////////////////// + // if no froms found, error, if more than one found, log a warning and use the first one // + /////////////////////////////////////////////////////////////////////////////////////////// + if(partyList.isEmpty()) + { + throw (new QException("Cannot send SES message because a FROM was not provided.")); + } + else if(partyList.size() > 1) + { + LOG.warn("More than one FROM value was found, will send using the first one found [" + partyList.get(0).getAddress() + "]."); + } + + ///////////////////////////// + // return the from address // + ///////////////////////////// + return (partyList.get(0).getAddress()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + List getPartyListFromParty(Party party) + { + ////////////////////////////////////////////// + // get all parties into one list of parties // + ////////////////////////////////////////////// + List partyList = new ArrayList<>(); + if(party != null) + { + if(party instanceof MultiParty toMultiParty) + { + partyList.addAll(toMultiParty.getPartyList()); + } + else + { + partyList.add(party); + } + } + return (partyList); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + Destination buildDestination(SendMessageInput input) throws QException + { + //////////////////////////////////////////////////////////////////// + // iterate over the parties putting it the proper party type list // + //////////////////////////////////////////////////////////////////// + List toList = new ArrayList<>(); + List ccList = new ArrayList<>(); + List bccList = new ArrayList<>(); + + List partyList = getPartyListFromParty(input.getTo()); + for(Party party : partyList) + { + if(EmailPartyRole.CC.equals(party.getRole())) + { + ccList.add(party.getAddress()); + } + else if(EmailPartyRole.BCC.equals(party.getRole())) + { + bccList.add(party.getAddress()); + } + else if(party.getRole() == null || PartyRole.Default.DEFAULT.equals(party.getRole()) || EmailPartyRole.TO.equals(party.getRole())) + { + toList.add(party.getAddress()); + } + else + { + throw (new QException("An unrecognized recipient role of [" + party.getRole() + "] was provided.")); + } + } + + ////////////////////////////////////////// + // if no to addresses, this is an error // + ////////////////////////////////////////// + if(toList.isEmpty()) + { + throw (new QException("Cannot send SES message because no TO addresses were provided.")); + } + + ///////////////////////////////////////////// + // build and return aws destination object // + ///////////////////////////////////////////// + return (new Destination() + .withToAddresses(toList) + .withCcAddresses(ccList) + .withBccAddresses(bccList)); + } + + + + /******************************************************************************* + ** Set the amazonSES object. + *******************************************************************************/ + public void setAmazonSES(AmazonSimpleEmailService amazonSES) + { + this.amazonSES = amazonSES; + } + + + + /******************************************************************************* + ** Internal accessor for the amazonSES object - should always use this, not the field. + *******************************************************************************/ + protected AmazonSimpleEmailService getAmazonSES(SendMessageInput sendMessageInput) + { + if(amazonSES == null) + { + SESMessagingProviderMetaData messagingProvider = (SESMessagingProviderMetaData) QContext.getQInstance().getMessagingProvider(sendMessageInput.getMessagingProviderName()); + + ///////////////////////////////////////////// + // get credentials and build an SES client // + ///////////////////////////////////////////// + BasicAWSCredentials credentials = new BasicAWSCredentials(messagingProvider.getAccessKey(), messagingProvider.getSecretKey()); + amazonSES = AmazonSimpleEmailServiceClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withRegion(messagingProvider.getRegion()).build(); + } + + return amazonSES; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java index 60463273..e8fc860b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java @@ -53,6 +53,7 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface // for type = TABLE // ////////////////////// private String tableName; + private String overrideIdField; private List searchFields; private List orderByFields; @@ -630,4 +631,35 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface qInstance.addPossibleValueSource(this); } + + + /******************************************************************************* + ** Getter for overrideIdField + *******************************************************************************/ + public String getOverrideIdField() + { + return (this.overrideIdField); + } + + + + /******************************************************************************* + ** Setter for overrideIdField + *******************************************************************************/ + public void setOverrideIdField(String overrideIdField) + { + this.overrideIdField = overrideIdField; + } + + + + /******************************************************************************* + ** Fluent setter for overrideIdField + *******************************************************************************/ + public QPossibleValueSource withOverrideIdField(String overrideIdField) + { + this.overrideIdField = overrideIdField; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java index 5f60e4b0..e14df8bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java @@ -155,4 +155,26 @@ public class AbstractProcessMetaDataBuilder return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public AbstractProcessMetaDataBuilder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AbstractProcessMetaDataBuilder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java index b9517d6d..d4bd7ff7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java @@ -36,6 +36,7 @@ public enum QComponentType RECORD_LIST, PROCESS_SUMMARY_RESULTS, GOOGLE_DRIVE_SELECT_FOLDER, + WIDGET, HTML; /////////////////////////////////////////////////////////////////////////// // keep these values in sync with QComponentType.ts in qqq-frontend-core // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java index 68b7c8db..9de7595a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java @@ -26,8 +26,12 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; +import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; /******************************************************************************* @@ -43,6 +47,8 @@ public class QFrontendStepMetaData extends QStepMetaData private List recordListFields; private Map formFieldMap; + private List helpContents; + /******************************************************************************* @@ -340,4 +346,61 @@ public class QFrontendStepMetaData extends QStepMetaData return (rs); } + + + /******************************************************************************* + ** Getter for helpContents + *******************************************************************************/ + public List getHelpContents() + { + return (this.helpContents); + } + + + + /******************************************************************************* + ** Setter for helpContents + *******************************************************************************/ + public void setHelpContents(List helpContents) + { + this.helpContents = helpContents; + } + + + + /******************************************************************************* + ** Fluent setter for helpContents + *******************************************************************************/ + public QFrontendStepMetaData withHelpContents(List helpContents) + { + this.helpContents = helpContents; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for adding 1 helpContent + *******************************************************************************/ + public QFrontendStepMetaData withHelpContent(QHelpContent helpContent) + { + if(this.helpContents == null) + { + this.helpContents = new ArrayList<>(); + } + + QInstanceHelpContentManager.putHelpContentInList(helpContent, this.helpContents); + return (this); + } + + + + /******************************************************************************* + ** remove a single helpContent based on its set of roles + *******************************************************************************/ + public void removeHelpContent(Set roles) + { + QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, this.helpContents); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 8a1a5eae..fc7c7687 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -60,10 +60,15 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi private List stepList; // these are the steps that are ran, by-default, in the order they are ran in private Map steps; // this is the full map of possible steps + private QBackendStepMetaData cancelStep; + private QIcon icon; private QScheduleMetaData schedule; + private VariantRunStrategy variantRunStrategy; + private String variantBackend; + private Map supplementalMetaData; @@ -671,4 +676,108 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi return (this); } + + + /******************************************************************************* + ** Getter for variantRunStrategy + *******************************************************************************/ + public VariantRunStrategy getVariantRunStrategy() + { + return (this.variantRunStrategy); + } + + + + /******************************************************************************* + ** Setter for variantRunStrategy + *******************************************************************************/ + public void setVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + this.variantRunStrategy = variantRunStrategy; + } + + + + /******************************************************************************* + ** Fluent setter for variantRunStrategy + *******************************************************************************/ + public QProcessMetaData withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + this.variantRunStrategy = variantRunStrategy; + return (this); + } + + + + /******************************************************************************* + ** Getter for variantBackend + *******************************************************************************/ + public String getVariantBackend() + { + return (this.variantBackend); + } + + + + /******************************************************************************* + ** Setter for variantBackend + *******************************************************************************/ + public void setVariantBackend(String variantBackend) + { + this.variantBackend = variantBackend; + } + + + + /******************************************************************************* + ** Fluent setter for variantBackend + *******************************************************************************/ + public QProcessMetaData withVariantBackend(String variantBackend) + { + this.variantBackend = variantBackend; + return (this); + } + + + + /******************************************************************************* + ** Getter for the full map of all steps (not the step list!) + ** + *******************************************************************************/ + public Map getAllSteps() + { + return steps; + } + + + + /******************************************************************************* + ** Getter for cancelStep + *******************************************************************************/ + public QBackendStepMetaData getCancelStep() + { + return (this.cancelStep); + } + + + + /******************************************************************************* + ** Setter for cancelStep + *******************************************************************************/ + public void setCancelStep(QBackendStepMetaData cancelStep) + { + this.cancelStep = cancelStep; + } + + + + /******************************************************************************* + ** Fluent setter for cancelStep + *******************************************************************************/ + public QProcessMetaData withCancelStep(QBackendStepMetaData cancelStep) + { + this.cancelStep = cancelStep; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java new file mode 100644 index 00000000..888cc3ae --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java @@ -0,0 +1,32 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.processes; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum VariantRunStrategy +{ + PARALLEL, + SERIAL +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java index 3825b9ff..6184db3f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java @@ -22,9 +22,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.queues; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; - - /******************************************************************************* ** Meta-data for an source of Amazon SQS queues (e.g, an aws account/credential ** set, with a common base URL). @@ -39,8 +36,6 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData private String region; private String baseURL; - private QScheduleMetaData schedule; - /******************************************************************************* @@ -201,38 +196,4 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData return (this); } - - - /******************************************************************************* - ** Getter for schedule - ** - *******************************************************************************/ - public QScheduleMetaData getSchedule() - { - return schedule; - } - - - - /******************************************************************************* - ** Setter for schedule - ** - *******************************************************************************/ - public void setSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - } - - - - /******************************************************************************* - ** Fluent setter for schedule - ** - *******************************************************************************/ - public SQSQueueProviderMetaData withSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportView.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportView.java index 1247ddfe..fa929c0d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportView.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportView.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.reporting; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -40,11 +41,11 @@ public class QReportView implements Cloneable private ReportType type; private String titleFormat; private List titleFields; - private List pivotFields; + private List summaryFields; - private boolean includeHeaderRow = true; - private boolean includeTotalRow = false; - private boolean includePivotSubTotals = false; + private boolean includeHeaderRow = true; + private boolean includeTotalRow = false; + private boolean includeSummarySubTotals = false; private List columns; private List orderByFields; @@ -52,6 +53,9 @@ public class QReportView implements Cloneable private QCodeReference recordTransformStep; private QCodeReference viewCustomizer; + private String pivotTableSourceViewName; + private PivotTableDefinition pivotTableDefinition; + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Note: This class is Cloneable - think about if new fields added here need deep-copied in the clone method! // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -297,34 +301,34 @@ public class QReportView implements Cloneable /******************************************************************************* - ** Getter for pivotFields + ** Getter for summaryFields ** *******************************************************************************/ - public List getPivotFields() + public List getSummaryFields() { - return pivotFields; + return summaryFields; } /******************************************************************************* - ** Setter for pivotFields + ** Setter for summaryFields ** *******************************************************************************/ - public void setPivotFields(List pivotFields) + public void setSummaryFields(List summaryFields) { - this.pivotFields = pivotFields; + this.summaryFields = summaryFields; } /******************************************************************************* - ** Fluent setter for pivotFields + ** Fluent setter for summaryFields ** *******************************************************************************/ - public QReportView withPivotFields(List pivotFields) + public QReportView withSummaryFields(List summaryFields) { - this.pivotFields = pivotFields; + this.summaryFields = summaryFields; return (this); } @@ -399,34 +403,34 @@ public class QReportView implements Cloneable /******************************************************************************* - ** Getter for pivotSubTotals + ** Getter for summarySubTotals ** *******************************************************************************/ - public boolean getIncludePivotSubTotals() + public boolean getIncludeSummarySubTotals() { - return includePivotSubTotals; + return includeSummarySubTotals; } /******************************************************************************* - ** Setter for pivotSubTotals + ** Setter for summarySubTotals ** *******************************************************************************/ - public void setIncludePivotSubTotals(boolean includePivotSubTotals) + public void setIncludeSummarySubTotals(boolean includeSummarySubTotals) { - this.includePivotSubTotals = includePivotSubTotals; + this.includeSummarySubTotals = includeSummarySubTotals; } /******************************************************************************* - ** Fluent setter for pivotSubTotals + ** Fluent setter for summarySubTotals ** *******************************************************************************/ - public QReportView withIncludePivotSubTotals(boolean pivotSubTotals) + public QReportView withIncludeSummarySubTotals(boolean summarySubTotals) { - this.includePivotSubTotals = pivotSubTotals; + this.includeSummarySubTotals = summarySubTotals; return (this); } @@ -602,9 +606,9 @@ public class QReportView implements Cloneable clone.setTitleFields(new ArrayList<>(titleFields)); } - if(pivotFields != null) + if(summaryFields != null) { - clone.setPivotFields(new ArrayList<>(pivotFields)); + clone.setSummaryFields(new ArrayList<>(summaryFields)); } if(columns != null) @@ -624,4 +628,67 @@ public class QReportView implements Cloneable throw new AssertionError(); } } + + + + /******************************************************************************* + ** Getter for pivotTableSourceViewName + *******************************************************************************/ + public String getPivotTableSourceViewName() + { + return (this.pivotTableSourceViewName); + } + + + + /******************************************************************************* + ** Setter for pivotTableSourceViewName + *******************************************************************************/ + public void setPivotTableSourceViewName(String pivotTableSourceViewName) + { + this.pivotTableSourceViewName = pivotTableSourceViewName; + } + + + + /******************************************************************************* + ** Fluent setter for pivotTableSourceViewName + *******************************************************************************/ + public QReportView withPivotTableSourceViewName(String pivotTableSourceViewName) + { + this.pivotTableSourceViewName = pivotTableSourceViewName; + return (this); + } + + + + /******************************************************************************* + ** Getter for pivotTableDefinition + *******************************************************************************/ + public PivotTableDefinition getPivotTableDefinition() + { + return (this.pivotTableDefinition); + } + + + + /******************************************************************************* + ** Setter for pivotTableDefinition + *******************************************************************************/ + public void setPivotTableDefinition(PivotTableDefinition pivotTableDefinition) + { + this.pivotTableDefinition = pivotTableDefinition; + } + + + + /******************************************************************************* + ** Fluent setter for pivotTableDefinition + *******************************************************************************/ + public QReportView withPivotTableDefinition(PivotTableDefinition pivotTableDefinition) + { + this.pivotTableDefinition = pivotTableDefinition; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/ReportType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/ReportType.java index 8492537a..22b5cb67 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/ReportType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/ReportType.java @@ -28,6 +28,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.reporting; public enum ReportType { TABLE, // e.g., raw data in tabular form. - SUMMARY, // e.g., summaries computed within QQQ - PIVOT // e.g., a true spreadsheet pivot. Not initially supported... + SUMMARY, // e.g., summaries computed within QQQ. + PIVOT // e.g., a true spreadsheet pivot. } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java index ece9019a..d474a013 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java @@ -22,29 +22,41 @@ package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + /******************************************************************************* ** Meta-data to define scheduled actions within QQQ. ** - ** Initially, only supports repeating jobs, either on a given # of seconds or millis. + ** Supports repeating jobs, either on a given # of seconds or millis, or cron + ** expressions (though cron may not be supported by all schedulers!) + ** ** Can also specify an initialDelay - e.g., to avoid all jobs starting up at the ** same moment. ** - ** In the future we most likely would want to allow cron strings to be added here. *******************************************************************************/ public class QScheduleMetaData { - public enum RunStrategy - {PARALLEL, SERIAL} - - + private String schedulerName; + private String description; private Integer repeatSeconds; private Integer repeatMillis; private Integer initialDelaySeconds; private Integer initialDelayMillis; - private RunStrategy variantRunStrategy; - private String variantBackend; + private String cronExpression; + private String cronTimeZoneId; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean isCron() + { + return StringUtils.hasContent(cronExpression); + } @@ -185,63 +197,125 @@ public class QScheduleMetaData /******************************************************************************* - ** Getter for variantBackend + ** Getter for cronExpression *******************************************************************************/ - public String getVariantBackend() + public String getCronExpression() { - return (this.variantBackend); + return (this.cronExpression); } /******************************************************************************* - ** Setter for variantBackend + ** Setter for cronExpression *******************************************************************************/ - public void setVariantBackend(String variantBackend) + public void setCronExpression(String cronExpression) { - this.variantBackend = variantBackend; + this.cronExpression = cronExpression; } /******************************************************************************* - ** Fluent setter for variantBackend + ** Fluent setter for cronExpression *******************************************************************************/ - public QScheduleMetaData withBackendVariant(String backendVariant) + public QScheduleMetaData withCronExpression(String cronExpression) { - this.variantBackend = backendVariant; + this.cronExpression = cronExpression; return (this); } /******************************************************************************* - ** Getter for variantRunStrategy + ** Getter for cronTimeZoneId *******************************************************************************/ - public RunStrategy getVariantRunStrategy() + public String getCronTimeZoneId() { - return (this.variantRunStrategy); + return (this.cronTimeZoneId); } /******************************************************************************* - ** Setter for variantRunStrategy + ** Setter for cronTimeZoneId *******************************************************************************/ - public void setVariantRunStrategy(RunStrategy variantRunStrategy) + public void setCronTimeZoneId(String cronTimeZoneId) { - this.variantRunStrategy = variantRunStrategy; + this.cronTimeZoneId = cronTimeZoneId; } /******************************************************************************* - ** Fluent setter for variantRunStrategy + ** Fluent setter for cronTimeZoneId *******************************************************************************/ - public QScheduleMetaData withVariantRunStrategy(RunStrategy variantRunStrategy) + public QScheduleMetaData withCronTimeZoneId(String cronTimeZoneId) { - this.variantRunStrategy = variantRunStrategy; + this.cronTimeZoneId = cronTimeZoneId; return (this); } + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public QScheduleMetaData withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + + /******************************************************************************* + ** Getter for description + *******************************************************************************/ + public String getDescription() + { + return (this.description); + } + + + + /******************************************************************************* + ** Setter for description + *******************************************************************************/ + public void setDescription(String description) + { + this.description = description; + } + + + + /******************************************************************************* + ** Fluent setter for description + *******************************************************************************/ + public QScheduleMetaData withDescription(String description) + { + this.description = description; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java new file mode 100644 index 00000000..9eda36b2 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; + + +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class QSchedulerMetaData implements TopLevelMetaDataInterface +{ + private String name; + private String type; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean supportsCronSchedules() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) throws QException; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public QSchedulerMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public String getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public QSchedulerMetaData withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addScheduler(this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java new file mode 100644 index 00000000..3367cd25 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java @@ -0,0 +1,130 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz; + + +import java.util.Properties; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzSchedulerMetaData extends QSchedulerMetaData +{ + private static final QLogger LOG = QLogger.getLogger(QuartzSchedulerMetaData.class); + + public static final String TYPE = "quartz"; + + private Properties properties; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QuartzSchedulerMetaData() + { + setType(TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsCronSchedules() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) throws QException + { + try + { + QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, getName(), getProperties(), systemSessionSupplier); + return (quartzScheduler); + } + catch(Exception e) + { + LOG.error("Error initializing quartz scheduler", e); + throw (new QException("Error initializing quartz scheduler", e)); + } + } + + + + /******************************************************************************* + ** Getter for properties + *******************************************************************************/ + public Properties getProperties() + { + return (this.properties); + } + + + + /******************************************************************************* + ** Setter for properties + *******************************************************************************/ + public void setProperties(Properties properties) + { + this.properties = properties; + } + + + + /******************************************************************************* + ** Fluent setter for properties + *******************************************************************************/ + public QuartzSchedulerMetaData withProperties(Properties properties) + { + this.properties = properties; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java new file mode 100644 index 00000000..69882b41 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple; + + +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.simple.SimpleScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SimpleSchedulerMetaData extends QSchedulerMetaData +{ + public static final String TYPE = "simple"; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SimpleSchedulerMetaData() + { + setType(TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsCronSchedules() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) + { + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + simpleScheduler.setSessionSupplier(systemSessionSupplier); + simpleScheduler.setSchedulerName(getName()); + return simpleScheduler; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/MultiRecordSecurityLock.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/MultiRecordSecurityLock.java new file mode 100644 index 00000000..04cb945b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/MultiRecordSecurityLock.java @@ -0,0 +1,198 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.security; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Subclass of RecordSecurityLock, for combining multiple locks using a boolean + ** (AND/OR) condition. Note that the combined locks can themselves also be + ** Multi-locks, thus creating a tree of locks. + *******************************************************************************/ +public class MultiRecordSecurityLock extends RecordSecurityLock implements Cloneable +{ + private List locks = new ArrayList<>(); + private BooleanOperator operator; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected MultiRecordSecurityLock clone() throws CloneNotSupportedException + { + MultiRecordSecurityLock clone = (MultiRecordSecurityLock) super.clone(); + + ///////////////////////// + // deep-clone the list // + ///////////////////////// + if(locks != null) + { + clone.locks = new ArrayList<>(); + for(RecordSecurityLock lock : locks) + { + clone.locks.add(lock.clone()); + } + } + + return (clone); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public enum BooleanOperator + { + AND, + OR; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QQueryFilter.BooleanOperator toFilterOperator() + { + return switch(this) + { + case AND -> QQueryFilter.BooleanOperator.AND; + case OR -> QQueryFilter.BooleanOperator.OR; + }; + } + } + + + + //////////////////////////////// + // todo - remove, this is POC // + //////////////////////////////// + static + { + new QTableMetaData() + .withName("savedReport") + .withRecordSecurityLock(new MultiRecordSecurityLock() + .withLocks(List.of( + new RecordSecurityLock() + .withFieldName("userId") + .withSecurityKeyType("user") + .withNullValueBehavior(NullValueBehavior.DENY) + .withLockScope(LockScope.READ_AND_WRITE), + new RecordSecurityLock() + .withFieldName("sharedReport.userId") + .withJoinNameChain(List.of("reportJoinSharedReport")) + .withSecurityKeyType("user") + .withNullValueBehavior(NullValueBehavior.DENY) + .withLockScope(LockScope.READ_AND_WRITE), // dynamic, from a value... + new RecordSecurityLock() + .withFieldName("sharedReport.groupId") + .withJoinNameChain(List.of("reportJoinSharedReport")) + .withSecurityKeyType("group") + .withNullValueBehavior(NullValueBehavior.DENY) + .withLockScope(LockScope.READ_AND_WRITE) // dynamic, from a value... + ))); + + } + + /******************************************************************************* + ** Getter for locks + *******************************************************************************/ + public List getLocks() + { + return (this.locks); + } + + + + /******************************************************************************* + ** Setter for locks + *******************************************************************************/ + public void setLocks(List locks) + { + this.locks = locks; + } + + + + /******************************************************************************* + ** Fluent setter for locks + *******************************************************************************/ + public MultiRecordSecurityLock withLocks(List locks) + { + this.locks = locks; + return (this); + } + + + + /******************************************************************************* + ** Fluently add one lock + *******************************************************************************/ + public MultiRecordSecurityLock withLock(RecordSecurityLock lock) + { + if(this.locks == null) + { + this.locks = new ArrayList<>(); + } + this.locks.add(lock); + return (this); + } + + + + /******************************************************************************* + ** Getter for operator + *******************************************************************************/ + public BooleanOperator getOperator() + { + return (this.operator); + } + + + + /******************************************************************************* + ** Setter for operator + *******************************************************************************/ + public void setOperator(BooleanOperator operator) + { + this.operator = operator; + } + + + + /******************************************************************************* + ** Fluent setter for operator + *******************************************************************************/ + public MultiRecordSecurityLock withOperator(BooleanOperator operator) + { + this.operator = operator; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLock.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLock.java index b16deac0..6cf99e9e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLock.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLock.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.security; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,7 +42,7 @@ import java.util.Map; ** - READ_AND_WRITE means that users cannot read or write records without a valid key. ** - WRITE means that users cannot write records without a valid key (but they can read them). *******************************************************************************/ -public class RecordSecurityLock +public class RecordSecurityLock implements Cloneable { private String securityKeyType; private String fieldName; @@ -52,6 +53,28 @@ public class RecordSecurityLock + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected RecordSecurityLock clone() throws CloneNotSupportedException + { + RecordSecurityLock clone = (RecordSecurityLock) super.clone(); + + ///////////////////////// + // deep-clone the list // + ///////////////////////// + if(joinNameChain != null) + { + clone.joinNameChain = new ArrayList<>(); + clone.joinNameChain.addAll(joinNameChain); + } + + return (clone); + } + + + /******************************************************************************* ** Constructor ** @@ -106,8 +129,9 @@ public class RecordSecurityLock *******************************************************************************/ public enum LockScope { - READ_AND_WRITE, - WRITE + READ_AND_WRITE, // lock both reads and writes + WRITE, // only lock writes + READ // only lock reads } @@ -265,4 +289,22 @@ public class RecordSecurityLock return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "RecordSecurityLock{" + + "securityKeyType='" + securityKeyType + '\'' + + ", fieldName='" + fieldName + '\'' + + ", joinNameChain=" + joinNameChain + + ", nullValueBehavior=" + nullValueBehavior + + ", lockScope=" + lockScope + + '}'; + } + } + diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFilters.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFilters.java index c8c7e9dc..c3eccc7e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFilters.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFilters.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.security; import java.util.List; +import java.util.Set; /******************************************************************************* @@ -46,6 +47,65 @@ public class RecordSecurityLockFilters + /******************************************************************************* + ** filter a list of locks so that we only see the ones that apply to reads. + *******************************************************************************/ + public static MultiRecordSecurityLock filterForReadLockTree(List recordSecurityLocks) + { + return filterForLockTree(recordSecurityLocks, Set.of(RecordSecurityLock.LockScope.READ_AND_WRITE, RecordSecurityLock.LockScope.READ)); + } + + + + /******************************************************************************* + ** filter a list of locks so that we only see the ones that apply to writes. + *******************************************************************************/ + public static MultiRecordSecurityLock filterForWriteLockTree(List recordSecurityLocks) + { + return filterForLockTree(recordSecurityLocks, Set.of(RecordSecurityLock.LockScope.READ_AND_WRITE, RecordSecurityLock.LockScope.WRITE)); + } + + + + /******************************************************************************* + ** filter a list of locks so that we only see the ones that apply to any of the + ** input set of scopes. + *******************************************************************************/ + private static MultiRecordSecurityLock filterForLockTree(List recordSecurityLocks, Set allowedScopes) + { + if(recordSecurityLocks == null) + { + return (null); + } + + ////////////////////////////////////////////////////////////// + // at the top-level we build a multi-lock with AND operator // + ////////////////////////////////////////////////////////////// + MultiRecordSecurityLock result = new MultiRecordSecurityLock(); + result.setOperator(MultiRecordSecurityLock.BooleanOperator.AND); + + for(RecordSecurityLock recordSecurityLock : recordSecurityLocks) + { + if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock) + { + MultiRecordSecurityLock filteredSubLock = filterForLockTree(multiRecordSecurityLock.getLocks(), allowedScopes); + filteredSubLock.setOperator(multiRecordSecurityLock.getOperator()); + result.withLock(filteredSubLock); + } + else + { + if(allowedScopes.contains(recordSecurityLock.getLockScope())) + { + result.withLock(recordSecurityLock); + } + } + } + + return (result); + } + + + /******************************************************************************* ** filter a list of locks so that we only see the ones that apply to writes. *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/QStepMetaDataDeserializer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/QStepMetaDataDeserializer.java index cc3ec609..5dc9afe4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/QStepMetaDataDeserializer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/QStepMetaDataDeserializer.java @@ -39,18 +39,21 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; *******************************************************************************/ public class QStepMetaDataDeserializer extends JsonDeserializer { + + /*************************************************************************** + ** + ***************************************************************************/ @Override - @SuppressWarnings("checkstyle:Indentation") public QStepMetaData deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException { TreeNode treeNode = jsonParser.readValueAsTree(); String stepType = DeserializerUtils.readTextValue(treeNode, "stepType"); Class targetClass = switch(stepType) - { - case "backend" -> QBackendStepMetaData.class; - case "frontend" -> QFrontendStepMetaData.class; - default -> throw new IllegalArgumentException("Unsupported StepType " + stepType + " for deserialization"); - }; + { + case "backend" -> QBackendStepMetaData.class; + case "frontend" -> QFrontendStepMetaData.class; + default -> throw new IllegalArgumentException("Unsupported StepType " + stepType + " for deserialization"); + }; return (DeserializerUtils.reflectivelyDeserialize(targetClass, treeNode)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareScopePossibleValueMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareScopePossibleValueMetaDataProducer.java new file mode 100644 index 00000000..49e5bb64 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareScopePossibleValueMetaDataProducer.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.sharing; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.processes.implementations.sharing.ShareScope; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ShareScopePossibleValueMetaDataProducer implements MetaDataProducerInterface +{ + public static final String NAME = "shareScope"; + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValueSource produce(QInstance qInstance) throws QException + { + return QPossibleValueSource.newForEnum(NAME, ShareScope.values()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableAudienceType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableAudienceType.java new file mode 100644 index 00000000..2ab36ff3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableAudienceType.java @@ -0,0 +1,186 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.sharing; + + +import java.io.Serializable; + + +/******************************************************************************* + ** As a component of a ShareableTableMetaData instance, define details about + ** one particular audience type. + ** + ** e.g., if a table can be shared to users and groups, there'd be 2 instances of + ** this object - one like: + ** - name: user + ** - fieldName: userId + ** - sourceTableName: User.TABLE_NAME + ** - sourceTableKeyFieldName: email (e.g., can be a UK, not just the PKey) + ** + ** and another similar, w/ the group-type details. + *******************************************************************************/ +public class ShareableAudienceType implements Serializable +{ + private String name; + private String fieldName; + private String sourceTableName; + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // maybe normally the primary key in the source table, but could be a unique-key instead sometimes // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + private String sourceTableKeyFieldName; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ShareableAudienceType() + { + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public ShareableAudienceType withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public ShareableAudienceType withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for sourceTableName + *******************************************************************************/ + public String getSourceTableName() + { + return (this.sourceTableName); + } + + + + /******************************************************************************* + ** Setter for sourceTableName + *******************************************************************************/ + public void setSourceTableName(String sourceTableName) + { + this.sourceTableName = sourceTableName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceTableName + *******************************************************************************/ + public ShareableAudienceType withSourceTableName(String sourceTableName) + { + this.sourceTableName = sourceTableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for sourceTableKeyFieldName + *******************************************************************************/ + public String getSourceTableKeyFieldName() + { + return (this.sourceTableKeyFieldName); + } + + + + /******************************************************************************* + ** Setter for sourceTableKeyFieldName + *******************************************************************************/ + public void setSourceTableKeyFieldName(String sourceTableKeyFieldName) + { + this.sourceTableKeyFieldName = sourceTableKeyFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceTableKeyFieldName + *******************************************************************************/ + public ShareableAudienceType withSourceTableKeyFieldName(String sourceTableKeyFieldName) + { + this.sourceTableKeyFieldName = sourceTableKeyFieldName; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableTableMetaData.java new file mode 100644 index 00000000..76c2afb6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableTableMetaData.java @@ -0,0 +1,398 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.sharing; + + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** meta data to attach to a table, to describe that its records are shareable. + *******************************************************************************/ +public class ShareableTableMetaData implements Serializable +{ + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // this is the name of the table that is a many-to-one join to the table whose records are being shared. // + // not the table whose records are shared (the asset table) // + // for example: given that we want to share "savedReports", the value here could be "sharedSavedReports" // + // and this object will be attached to the savedReports table. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + private String sharedRecordTableName; + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // name of the field in the sharedRecordTable that has a foreign key pointing at the asset table // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + private String assetIdFieldName; + + ////////////////////////////////////////////////////// + // name of the scope field in the sharedRecordTable // + ////////////////////////////////////////////////////// + private String scopeFieldName; + + /////////////////////////////////////////////////////////// + // map of audienceTypes names to type definition objects // + /////////////////////////////////////////////////////////// + private Map audienceTypes; + + ///////////////////////////////////////////////// + // PVS that lists the available audience types // + ///////////////////////////////////////////////// + private String audienceTypesPossibleValueSourceName; + + /////////////////////////////////////////////////// + // PVS that lists the available audience records // + /////////////////////////////////////////////////// + private String audiencePossibleValueSourceName; + + ////////////////////////////////////////////////////////////// + // name of a field in "this" table, that has the owner's id // + ////////////////////////////////////////////////////////////// + private String thisTableOwnerIdFieldName; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ShareableTableMetaData() + { + } + + + + /******************************************************************************* + ** Getter for sharedRecordTableName + *******************************************************************************/ + public String getSharedRecordTableName() + { + return (this.sharedRecordTableName); + } + + + + /******************************************************************************* + ** Setter for sharedRecordTableName + *******************************************************************************/ + public void setSharedRecordTableName(String sharedRecordTableName) + { + this.sharedRecordTableName = sharedRecordTableName; + } + + + + /******************************************************************************* + ** Fluent setter for sharedRecordTableName + *******************************************************************************/ + public ShareableTableMetaData withSharedRecordTableName(String sharedRecordTableName) + { + this.sharedRecordTableName = sharedRecordTableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for assetIdFieldName + *******************************************************************************/ + public String getAssetIdFieldName() + { + return (this.assetIdFieldName); + } + + + + /******************************************************************************* + ** Setter for assetIdFieldName + *******************************************************************************/ + public void setAssetIdFieldName(String assetIdFieldName) + { + this.assetIdFieldName = assetIdFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for assetIdFieldName + *******************************************************************************/ + public ShareableTableMetaData withAssetIdFieldName(String assetIdFieldName) + { + this.assetIdFieldName = assetIdFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for scopeFieldName + *******************************************************************************/ + public String getScopeFieldName() + { + return (this.scopeFieldName); + } + + + + /******************************************************************************* + ** Setter for scopeFieldName + *******************************************************************************/ + public void setScopeFieldName(String scopeFieldName) + { + this.scopeFieldName = scopeFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for scopeFieldName + *******************************************************************************/ + public ShareableTableMetaData withScopeFieldName(String scopeFieldName) + { + this.scopeFieldName = scopeFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for audienceTypes + *******************************************************************************/ + public Map getAudienceTypes() + { + return (this.audienceTypes); + } + + + + /******************************************************************************* + ** Setter for audienceTypes + *******************************************************************************/ + public void setAudienceTypes(Map audienceTypes) + { + this.audienceTypes = audienceTypes; + } + + + + /******************************************************************************* + ** Fluent setter for audienceTypes + *******************************************************************************/ + public ShareableTableMetaData withAudienceTypes(Map audienceTypes) + { + this.audienceTypes = audienceTypes; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for audienceTypes + *******************************************************************************/ + public ShareableTableMetaData withAudienceType(ShareableAudienceType audienceType) + { + if(this.audienceTypes == null) + { + this.audienceTypes = new LinkedHashMap<>(); + } + + if(audienceType.getName() == null) + { + throw (new IllegalArgumentException("Attempt to add an audience type without a name")); + } + + if(this.audienceTypes.containsKey(audienceType.getName())) + { + throw (new IllegalArgumentException("Attempt to add more than 1 audience type with the same name [" + audienceType.getName() + "]")); + } + + this.audienceTypes.put(audienceType.getName(), audienceType); + + return (this); + } + + + + /******************************************************************************* + ** Getter for audienceTypesPossibleValueSourceName + *******************************************************************************/ + public String getAudienceTypesPossibleValueSourceName() + { + return (this.audienceTypesPossibleValueSourceName); + } + + + + /******************************************************************************* + ** Setter for audienceTypesPossibleValueSourceName + *******************************************************************************/ + public void setAudienceTypesPossibleValueSourceName(String audienceTypesPossibleValueSourceName) + { + this.audienceTypesPossibleValueSourceName = audienceTypesPossibleValueSourceName; + } + + + + /******************************************************************************* + ** Fluent setter for audienceTypesPossibleValueSourceName + *******************************************************************************/ + public ShareableTableMetaData withAudienceTypesPossibleValueSourceName(String audienceTypesPossibleValueSourceName) + { + this.audienceTypesPossibleValueSourceName = audienceTypesPossibleValueSourceName; + return (this); + } + + + + /******************************************************************************* + ** Getter for thisTableOwnerIdFieldName + *******************************************************************************/ + public String getThisTableOwnerIdFieldName() + { + return (this.thisTableOwnerIdFieldName); + } + + + + /******************************************************************************* + ** Setter for thisTableOwnerIdFieldName + *******************************************************************************/ + public void setThisTableOwnerIdFieldName(String thisTableOwnerIdFieldName) + { + this.thisTableOwnerIdFieldName = thisTableOwnerIdFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for thisTableOwnerIdFieldName + *******************************************************************************/ + public ShareableTableMetaData withThisTableOwnerIdFieldName(String thisTableOwnerIdFieldName) + { + this.thisTableOwnerIdFieldName = thisTableOwnerIdFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for audiencePossibleValueSourceName + *******************************************************************************/ + public String getAudiencePossibleValueSourceName() + { + return (this.audiencePossibleValueSourceName); + } + + + + /******************************************************************************* + ** Setter for audiencePossibleValueSourceName + *******************************************************************************/ + public void setAudiencePossibleValueSourceName(String audiencePossibleValueSourceName) + { + this.audiencePossibleValueSourceName = audiencePossibleValueSourceName; + } + + + + /******************************************************************************* + ** Fluent setter for audiencePossibleValueSourceName + *******************************************************************************/ + public ShareableTableMetaData withAudiencePossibleValueSourceName(String audiencePossibleValueSourceName) + { + this.audiencePossibleValueSourceName = audiencePossibleValueSourceName; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void validate(QInstance qInstance, QTableMetaData tableMetaData, QInstanceValidator qInstanceValidator) + { + String prefix = "ShareableTableMetaData for table [" + tableMetaData.getName() + "]: "; + if(qInstanceValidator.assertCondition(StringUtils.hasContent(sharedRecordTableName), prefix + "missing sharedRecordTableName.")) + { + boolean hasAssetIdFieldName = qInstanceValidator.assertCondition(StringUtils.hasContent(assetIdFieldName), prefix + "missing assetIdFieldName"); + boolean hasScopeFieldName = qInstanceValidator.assertCondition(StringUtils.hasContent(scopeFieldName), prefix + "missing scopeFieldName"); + + QTableMetaData sharedRecordTable = qInstance.getTable(sharedRecordTableName); + boolean hasValidSharedRecordTable = qInstanceValidator.assertCondition(sharedRecordTable != null, prefix + "unrecognized sharedRecordTableName [" + sharedRecordTableName + "]"); + + if(hasValidSharedRecordTable && hasAssetIdFieldName) + { + qInstanceValidator.assertCondition(sharedRecordTable.getFields().containsKey(assetIdFieldName), prefix + "unrecognized assertIdFieldName [" + assetIdFieldName + "] in sharedRecordTable [" + sharedRecordTableName + "]"); + } + + if(hasValidSharedRecordTable && hasScopeFieldName) + { + qInstanceValidator.assertCondition(sharedRecordTable.getFields().containsKey(scopeFieldName), prefix + "unrecognized scopeFieldName [" + scopeFieldName + "] in sharedRecordTable [" + sharedRecordTableName + "]"); + } + + if(qInstanceValidator.assertCondition(CollectionUtils.nullSafeHasContents(audienceTypes), prefix + "missing audienceTypes")) + { + for(Map.Entry entry : audienceTypes.entrySet()) + { + ShareableAudienceType audienceType = entry.getValue(); + qInstanceValidator.assertCondition(Objects.equals(entry.getKey(), audienceType.getName()), prefix + "inconsistent naming for shareableAudienceType [" + entry.getKey() + "] != [" + audienceType.getName() + "]"); + if(qInstanceValidator.assertCondition(StringUtils.hasContent(audienceType.getFieldName()), prefix + "missing fieldName for shareableAudienceType [" + entry.getKey() + "]") && hasValidSharedRecordTable) + { + qInstanceValidator.assertCondition(sharedRecordTable.getFields().containsKey(audienceType.getFieldName()), prefix + "unrecognized fieldName [" + audienceType.getFieldName() + "] for shareableAudienceType [" + entry.getKey() + "] in sharedRecordTable [" + sharedRecordTableName + "]"); + } + + // todo - validate this audienceType.getSourceTableKeyFieldName() is a field, and it is a UKey + + /* todo - make these optional i guess, because i didn't put user table in qqq + boolean hasSourceTableKeyFieldName = qInstanceValidator.assertCondition(StringUtils.hasContent(audienceType.getSourceTableKeyFieldName()), prefix + "missing sourceTableKeyFieldName for shareableAudienceType [" + entry.getKey() + "]"); + if(qInstanceValidator.assertCondition(qInstance.getTable(audienceType.getSourceTableName()) != null, prefix + "unrecognized sourceTableName [" + audienceType.getSourceTableName() + "] for shareableAudienceType [" + entry.getKey() + "] in sharedRecordTable [" + sharedRecordTableName + "]") && hasSourceTableKeyFieldName) + { + qInstanceValidator.assertCondition(qInstance.getTable(audienceType.getSourceTableName()).getFields().containsKey(audienceType.getSourceTableKeyFieldName()), prefix + "unrecognized sourceTableKeyFieldName [" + audienceType.getSourceTableKeyFieldName() + "] for shareableAudienceType [" + entry.getKey() + "] in sharedRecordTable [" + sharedRecordTableName + "]"); + } + */ + } + } + } + + if(StringUtils.hasContent(thisTableOwnerIdFieldName)) + { + qInstanceValidator.assertCondition(tableMetaData.getFields().containsKey(thisTableOwnerIdFieldName), prefix + "unrecognized thisTableOwnerIdFieldName [" + thisTableOwnerIdFieldName + "]"); + } + + if(StringUtils.hasContent(audienceTypesPossibleValueSourceName)) + { + qInstanceValidator.assertCondition(qInstance.getPossibleValueSource(audienceTypesPossibleValueSourceName) != null, prefix + "unrecognized audienceTypesPossibleValueSourceName [" + audienceTypesPossibleValueSourceName + "]"); + } + + if(StringUtils.hasContent(audiencePossibleValueSourceName)) + { + qInstanceValidator.assertCondition(qInstance.getPossibleValueSource(audiencePossibleValueSourceName) != null, prefix + "unrecognized audiencePossibleValueSourceName [" + audiencePossibleValueSourceName + "]"); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java index e5e39c73..f272620f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java @@ -22,6 +22,10 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; +import java.util.HashSet; +import java.util.Set; + + /******************************************************************************* ** Things that can be done to tables, fields. ** @@ -38,5 +42,26 @@ public enum Capability // keep these values in sync with Capability.ts in qqq-frontend-core // /////////////////////////////////////////////////////////////////////// - QUERY_STATS + QUERY_STATS; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Set allReadCapabilities() + { + return (new HashSet<>(Set.of(TABLE_QUERY, TABLE_GET, TABLE_COUNT, QUERY_STATS))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Set allWriteCapabilities() + { + return (new HashSet<>(Set.of(TABLE_INSERT, TABLE_UPDATE, TABLE_DELETE))); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index fac05aaa..9819f5b3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -107,6 +108,8 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData private List exposedJoins; + private ShareableTableMetaData shareableTableMetaData; + /******************************************************************************* @@ -156,11 +159,26 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData public QTableMetaData withFieldsFromEntity(Class entityClass) throws QException { List recordEntityFieldList = QRecordEntity.getFieldList(entityClass); + + boolean setPrimaryKey = false; + for(QRecordEntityField recordEntityField : recordEntityFieldList) { QFieldMetaData field = new QFieldMetaData(recordEntityField.getGetter()); addField(field); + + if(recordEntityField.getFieldAnnotation().isPrimaryKey()) + { + if(setPrimaryKey) + { + throw (new QException("Attempt to set more than one field as primary key (" + primaryKeyField + "," + field.getName() + ").")); + } + + setPrimaryKeyField(field.getName()); + setPrimaryKey = true; + } } + return (this); } @@ -622,6 +640,18 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData + /******************************************************************************* + ** fluent setter for both recordLabelFormat and recordLabelFields + *******************************************************************************/ + public QTableMetaData withRecordLabelFormatAndFields(String format, String... fields) + { + setRecordLabelFormat(format); + setRecordLabelFields(Arrays.asList(fields)); + return (this); + } + + + /******************************************************************************* ** Getter for recordLabelFields ** @@ -1385,4 +1415,35 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData return (this); } + + + /******************************************************************************* + ** Getter for shareableTableMetaData + *******************************************************************************/ + public ShareableTableMetaData getShareableTableMetaData() + { + return (this.shareableTableMetaData); + } + + + + /******************************************************************************* + ** Setter for shareableTableMetaData + *******************************************************************************/ + public void setShareableTableMetaData(ShareableTableMetaData shareableTableMetaData) + { + this.shareableTableMetaData = shareableTableMetaData; + } + + + + /******************************************************************************* + ** Fluent setter for shareableTableMetaData + *******************************************************************************/ + public QTableMetaData withShareableTableMetaData(ShareableTableMetaData shareableTableMetaData) + { + this.shareableTableMetaData = shareableTableMetaData; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java index 303e2932..13a57662 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; /******************************************************************************* @@ -37,6 +38,8 @@ public class QTableAutomationDetails private Integer overrideBatchSize; + private QScheduleMetaData schedule; + private String shardByFieldName; // field in "this" table, to use for sharding private String shardSourceTableName; // name of the table where the shards are defined as rows private String shardLabelFieldName; // field in shard-source-table to use for labeling shards @@ -317,4 +320,35 @@ public class QTableAutomationDetails return (this); } + + /******************************************************************************* + ** Getter for schedule + *******************************************************************************/ + public QScheduleMetaData getSchedule() + { + return (this.schedule); + } + + + + /******************************************************************************* + ** Setter for schedule + *******************************************************************************/ + public void setSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + } + + + + /******************************************************************************* + ** Fluent setter for schedule + *******************************************************************************/ + public QTableAutomationDetails withSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStat.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStat.java index a0e5f513..ef14ac72 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStat.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStat.java @@ -28,6 +28,7 @@ import java.util.Set; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QAssociation; import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QIgnore; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; @@ -77,7 +78,9 @@ public class QueryStat extends QRecordEntity /////////////////////////////////////////////////////////// // non-persistent fields - used to help build the record // /////////////////////////////////////////////////////////// - private String tableName; + @QIgnore + private String tableName; + private Set joinTableNames; private QQueryFilter queryFilter; @@ -384,6 +387,7 @@ public class QueryStat extends QRecordEntity /******************************************************************************* ** Getter for queryFilter *******************************************************************************/ + @QIgnore public QQueryFilter getQueryFilter() { return (this.queryFilter); @@ -446,6 +450,7 @@ public class QueryStat extends QRecordEntity /******************************************************************************* ** Getter for joinTableNames *******************************************************************************/ + @QIgnore public Set getJoinTableNames() { return (this.joinTableNames); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java index 62e460bf..c9cbd826 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java @@ -65,6 +65,7 @@ public class QueryStatMetaDataProvider instance.addTable(defineStandardTable(QueryStatJoinTable.TABLE_NAME, QueryStatJoinTable.class, backendName, backendDetailEnricher)); instance.addTable(defineStandardTable(QueryStatCriteriaField.TABLE_NAME, QueryStatCriteriaField.class, backendName, backendDetailEnricher) + .withIcon(new QIcon().withName("filter_alt")) .withExposedJoin(new ExposedJoin().withJoinTable(QueryStat.TABLE_NAME)) ); @@ -115,6 +116,7 @@ public class QueryStatMetaDataProvider QTableMetaData table = new QTableMetaData() .withName(QueryStat.TABLE_NAME) + .withIcon(new QIcon().withName("query_stats")) .withBackendName(backendName) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE)) .withRecordLabelFormat("%s") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReport.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReport.java new file mode 100644 index 00000000..19ef870d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReport.java @@ -0,0 +1,506 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** Entity bean for the rendered report table + *******************************************************************************/ +public class RenderedReport extends QRecordEntity +{ + public static final String TABLE_NAME = "renderedReport"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID) + private String userId; + + @QField(possibleValueSourceName = SavedReport.TABLE_NAME) + private Integer savedReportId; + + @QField(possibleValueSourceName = RenderedReportStatus.NAME, label = "Status") + private Integer renderedReportStatusId; + + @QField(maxLength = 40, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String jobUuid; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String resultPath; + + @QField(maxLength = 10, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ReportFormatPossibleValueEnum.NAME) + private String reportFormat; + + @QField() + private Instant startTime; + + @QField() + private Instant endTime; + + @QField(displayFormat = DisplayFormat.COMMAS) + private Integer rowCount; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS) + private String errorMessage; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RenderedReport() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RenderedReport(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public RenderedReport withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public RenderedReport withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public RenderedReport withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public RenderedReport withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for savedReportId + *******************************************************************************/ + public Integer getSavedReportId() + { + return (this.savedReportId); + } + + + + /******************************************************************************* + ** Setter for savedReportId + *******************************************************************************/ + public void setSavedReportId(Integer savedReportId) + { + this.savedReportId = savedReportId; + } + + + + /******************************************************************************* + ** Fluent setter for savedReportId + *******************************************************************************/ + public RenderedReport withSavedReportId(Integer savedReportId) + { + this.savedReportId = savedReportId; + return (this); + } + + + + /******************************************************************************* + ** Getter for renderedReportStatusId + *******************************************************************************/ + public Integer getRenderedReportStatusId() + { + return (this.renderedReportStatusId); + } + + + + /******************************************************************************* + ** Setter for renderedReportStatusId + *******************************************************************************/ + public void setRenderedReportStatusId(Integer renderedReportStatusId) + { + this.renderedReportStatusId = renderedReportStatusId; + } + + + + /******************************************************************************* + ** Fluent setter for renderedReportStatusId + *******************************************************************************/ + public RenderedReport withRenderedReportStatusId(Integer renderedReportStatusId) + { + this.renderedReportStatusId = renderedReportStatusId; + return (this); + } + + + + /******************************************************************************* + ** Getter for jobUuid + *******************************************************************************/ + public String getJobUuid() + { + return (this.jobUuid); + } + + + + /******************************************************************************* + ** Setter for jobUuid + *******************************************************************************/ + public void setJobUuid(String jobUuid) + { + this.jobUuid = jobUuid; + } + + + + /******************************************************************************* + ** Fluent setter for jobUuid + *******************************************************************************/ + public RenderedReport withJobUuid(String jobUuid) + { + this.jobUuid = jobUuid; + return (this); + } + + + + /******************************************************************************* + ** Getter for resultPath + *******************************************************************************/ + public String getResultPath() + { + return (this.resultPath); + } + + + + /******************************************************************************* + ** Setter for resultPath + *******************************************************************************/ + public void setResultPath(String resultPath) + { + this.resultPath = resultPath; + } + + + + /******************************************************************************* + ** Fluent setter for resultPath + *******************************************************************************/ + public RenderedReport withResultPath(String resultPath) + { + this.resultPath = resultPath; + return (this); + } + + + + /******************************************************************************* + ** Getter for reportFormat + *******************************************************************************/ + public String getReportFormat() + { + return (this.reportFormat); + } + + + + /******************************************************************************* + ** Setter for reportFormat + *******************************************************************************/ + public void setReportFormat(String reportFormat) + { + this.reportFormat = reportFormat; + } + + + + /******************************************************************************* + ** Fluent setter for reportFormat + *******************************************************************************/ + public RenderedReport withReportFormat(String reportFormat) + { + this.reportFormat = reportFormat; + return (this); + } + + + + /******************************************************************************* + ** Getter for startTime + *******************************************************************************/ + public Instant getStartTime() + { + return (this.startTime); + } + + + + /******************************************************************************* + ** Setter for startTime + *******************************************************************************/ + public void setStartTime(Instant startTime) + { + this.startTime = startTime; + } + + + + /******************************************************************************* + ** Fluent setter for startTime + *******************************************************************************/ + public RenderedReport withStartTime(Instant startTime) + { + this.startTime = startTime; + return (this); + } + + + + /******************************************************************************* + ** Getter for endTime + *******************************************************************************/ + public Instant getEndTime() + { + return (this.endTime); + } + + + + /******************************************************************************* + ** Setter for endTime + *******************************************************************************/ + public void setEndTime(Instant endTime) + { + this.endTime = endTime; + } + + + + /******************************************************************************* + ** Fluent setter for endTime + *******************************************************************************/ + public RenderedReport withEndTime(Instant endTime) + { + this.endTime = endTime; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowCount + *******************************************************************************/ + public Integer getRowCount() + { + return (this.rowCount); + } + + + + /******************************************************************************* + ** Setter for rowCount + *******************************************************************************/ + public void setRowCount(Integer rowCount) + { + this.rowCount = rowCount; + } + + + + /******************************************************************************* + ** Fluent setter for rowCount + *******************************************************************************/ + public RenderedReport withRowCount(Integer rowCount) + { + this.rowCount = rowCount; + return (this); + } + + + + /******************************************************************************* + ** Getter for errorMessage + *******************************************************************************/ + public String getErrorMessage() + { + return (this.errorMessage); + } + + + + /******************************************************************************* + ** Setter for errorMessage + *******************************************************************************/ + public void setErrorMessage(String errorMessage) + { + this.errorMessage = errorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for errorMessage + *******************************************************************************/ + public RenderedReport withErrorMessage(String errorMessage) + { + this.errorMessage = errorMessage; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReportStatus.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReportStatus.java new file mode 100644 index 00000000..b407fc07 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReportStatus.java @@ -0,0 +1,96 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum RenderedReportStatus implements PossibleValueEnum +{ + RUNNING(1, "Running"), + COMPLETE(2, "Complete"), + FAILED(3, "Failed"); + + public static final String NAME = "renderedReportStatus"; + + private final Integer id; + private final String label; + + + + /******************************************************************************* + ** + *******************************************************************************/ + RenderedReportStatus(int id, String label) + { + this.id = id; + this.label = label; + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Integer getPossibleValueId() + { + return id; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return label; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumn.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumn.java new file mode 100644 index 00000000..44518fe2 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumn.java @@ -0,0 +1,98 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.Serializable; + + +/******************************************************************************* + ** single entry in ReportColumns object - as part of SavedReport + *******************************************************************************/ +public class ReportColumn implements Serializable +{ + private String name; + private Boolean isVisible; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public ReportColumn withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for isVisible + *******************************************************************************/ + public Boolean getIsVisible() + { + return (this.isVisible); + } + + + + /******************************************************************************* + ** Setter for isVisible + *******************************************************************************/ + public void setIsVisible(Boolean isVisible) + { + this.isVisible = isVisible; + } + + + + /******************************************************************************* + ** Fluent setter for isVisible + *******************************************************************************/ + public ReportColumn withIsVisible(Boolean isVisible) + { + this.isVisible = isVisible; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumns.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumns.java new file mode 100644 index 00000000..a18e1dd9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumns.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** type of object expected to be in the SavedReport columnsJSON field + *******************************************************************************/ +public class ReportColumns implements Serializable +{ + private List columns; + + + + /******************************************************************************* + ** Getter for columns + *******************************************************************************/ + public List getColumns() + { + return (this.columns); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public List extractVisibleColumns() + { + return CollectionUtils.nonNullList(getColumns()).stream() + ////////////////////////////////////////////////////// + // if isVisible is missing, we assume it to be true // + ////////////////////////////////////////////////////// + .filter(rc -> rc.getIsVisible() == null || rc.getIsVisible()) + .filter(rc -> StringUtils.hasContent(rc.getName())) + .filter(rc -> !rc.getName().startsWith("__check")) + .toList(); + } + + + + /******************************************************************************* + ** Setter for columns + *******************************************************************************/ + public void setColumns(List columns) + { + this.columns = columns; + } + + + + /******************************************************************************* + ** Fluent setter for columns + *******************************************************************************/ + public ReportColumns withColumns(List columns) + { + this.columns = columns; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter to add 1 column + *******************************************************************************/ + public ReportColumns withColumn(ReportColumn column) + { + if(this.columns == null) + { + this.columns = new ArrayList<>(); + } + this.columns.add(column); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter to add 1 column w/ just a name + *******************************************************************************/ + public ReportColumns withColumn(String name) + { + if(this.columns == null) + { + this.columns = new ArrayList<>(); + } + this.columns.add(new ReportColumn().withName(name)); + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java new file mode 100644 index 00000000..fc90fb1f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java @@ -0,0 +1,205 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.FilterVariableExpression; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.DynamicFormWidgetData; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.SavedReportToReportMetaDataAdapter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.json.JSONObject; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Note - exists under 2 names, for the RenderSavedReport process, and for the + ** ScheduledReport table + *******************************************************************************/ +public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRenderer +{ + private static final QLogger LOG = QLogger.getLogger(ReportValuesDynamicFormWidgetRenderer.class); + + private QPossibleValueTranslator qPossibleValueTranslator; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + try + { + List fieldList = new ArrayList<>(); + Map defaultValues = new HashMap<>(); + + ////////////////////////////////////////////////////////////////////////////// + // read params to ultimately find the query filter that has variables in it // + ////////////////////////////////////////////////////////////////////////////// + SavedReport savedReport = null; + if(input.getQueryParams().containsKey("savedReportId")) + { + QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get("savedReportId")))); + savedReport = new SavedReport(record); + } + else if(input.getQueryParams().containsKey("id")) + { + QRecord scheduledReportRecord = new GetAction().executeForRecord(new GetInput(ScheduledReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get("id")))); + QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(scheduledReportRecord.getValueInteger("savedReportId")))); + savedReport = new SavedReport(record); + + String inputValues = scheduledReportRecord.getValueString("inputValues"); + if(StringUtils.hasContent(inputValues)) + { + JSONObject jsonObject = JsonUtils.toJSONObject(inputValues); + for(String key : jsonObject.keySet()) + { + defaultValues.put(key, jsonObject.optString(key)); + } + } + } + else + { + ////////////////////////////////// + // return quietly w/ nothing... // + ////////////////////////////////// + DynamicFormWidgetData widgetData = new DynamicFormWidgetData(); + return new RenderWidgetOutput(widgetData); + } + + QRecord recordOfFieldValues = new QRecord(); + + if(StringUtils.hasContent(savedReport.getQueryFilterJson())) + { + QQueryFilter queryFilter = SavedReportToReportMetaDataAdapter.getQQueryFilter(savedReport.getQueryFilterJson()); + QTableMetaData table = QContext.getQInstance().getTable(savedReport.getTableName()); + + /////////////////////////////////////////////////////////////////////////////////////////////// + // find variables in the query filter; convert them to a list of fields for the dynamic form // + /////////////////////////////////////////////////////////////////////////////////////////////// + for(QFilterCriteria criteria : CollectionUtils.nonNullList(queryFilter.getCriteria())) + { + for(Serializable criteriaValue : CollectionUtils.nonNullList(criteria.getValues())) + { + if(criteriaValue instanceof FilterVariableExpression filterVariableExpression) + { + GenerateReportAction.FieldAndJoinTable fieldAndJoinTable = GenerateReportAction.getFieldAndJoinTable(table, criteria.getFieldName()); + QFieldMetaData fieldMetaData = fieldAndJoinTable.field().clone(); + + ///////////////////////////////// + // make name & label for field // + ///////////////////////////////// + String fieldName = filterVariableExpression.getVariableName(); + fieldMetaData.setName(fieldName); + fieldMetaData.setLabel(QInstanceEnricher.nameToLabel(filterVariableExpression.getVariableName())); + + //////////////////////////////////////////////////////////// + // in this use case, every field is required and editable // + //////////////////////////////////////////////////////////// + fieldMetaData.setIsRequired(true); + fieldMetaData.setIsEditable(true); + + /////////////////////////////////////////////////////////////////////// + // if we're in a context where there are values, then populate those // + // e.g., a view screen instead of an edit screen, i think // + /////////////////////////////////////////////////////////////////////// + if(defaultValues.containsKey(fieldName)) + { + String value = defaultValues.get(fieldName); + + fieldMetaData.setDefaultValue(value); + recordOfFieldValues.setValue(fieldName, value); + + ////////////////////////////////////////////////////// + // look up display values for possible value fields // + ////////////////////////////////////////////////////// + if(StringUtils.hasContent(fieldMetaData.getPossibleValueSourceName())) + { + if(qPossibleValueTranslator == null) + { + qPossibleValueTranslator = new QPossibleValueTranslator(); + } + String displayValue = qPossibleValueTranslator.translatePossibleValue(fieldMetaData, value); + recordOfFieldValues.setDisplayValue(fieldName, displayValue); + } + } + + fieldList.add(fieldMetaData); + } + } + } + } + + /////////////////////////////////// + // make output object and return // + /////////////////////////////////// + DynamicFormWidgetData widgetData = new DynamicFormWidgetData(); + widgetData.setFieldList(fieldList); + widgetData.setRecordOfFieldValues(recordOfFieldValues); + widgetData.setMergedDynamicFormValuesIntoFieldName("inputValues"); + + if(CollectionUtils.nullSafeIsEmpty(fieldList)) + { + /////////////////////////////////////////////// + // actually don't show this for process mode // + /////////////////////////////////////////////// + if(!input.getQueryParams().containsKey("processName")) + { + widgetData.setNoFieldsMessage("This Report does not use any Variable Values"); + } + } + + return new RenderWidgetOutput(widgetData); + } + catch(Exception e) + { + LOG.warn("Error rendering scheduled report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams()))); + throw (new QException("Error rendering scheduled report values dynamic form widget", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReport.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReport.java new file mode 100644 index 00000000..1fd531d1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReport.java @@ -0,0 +1,386 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; + + +/******************************************************************************* + ** Entity bean for the saved report table + *******************************************************************************/ +public class SavedReport extends QRecordEntity +{ + public static final String TABLE_NAME = "savedReport"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS, label = "Report Name") + private String label; + + @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, label = "Table", isRequired = true) + private String tableName; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID, label = "Owner") + private String userId; + + @QField(label = "Query Filter") + private String queryFilterJson; + + @QField(label = "Columns") + private String columnsJson; + + @QField(label = "Input Fields") + private String inputFieldsJson; + + @QField(label = "Pivot Table") + private String pivotTableJson; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedReport() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedReport(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Setter for id + ** + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Getter for createDate + ** + *******************************************************************************/ + public Instant getCreateDate() + { + return createDate; + } + + + + /******************************************************************************* + ** Setter for createDate + ** + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Getter for modifyDate + ** + *******************************************************************************/ + public Instant getModifyDate() + { + return modifyDate; + } + + + + /******************************************************************************* + ** Setter for modifyDate + ** + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public SavedReport withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Setter for tableName + ** + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + ** + *******************************************************************************/ + public SavedReport withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + ** + *******************************************************************************/ + public String getUserId() + { + return userId; + } + + + + /******************************************************************************* + ** Setter for userId + ** + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + ** + *******************************************************************************/ + public SavedReport withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for queryFilterJson + *******************************************************************************/ + public String getQueryFilterJson() + { + return (this.queryFilterJson); + } + + + + /******************************************************************************* + ** Setter for queryFilterJson + *******************************************************************************/ + public void setQueryFilterJson(String queryFilterJson) + { + this.queryFilterJson = queryFilterJson; + } + + + + /******************************************************************************* + ** Fluent setter for queryFilterJson + *******************************************************************************/ + public SavedReport withQueryFilterJson(String queryFilterJson) + { + this.queryFilterJson = queryFilterJson; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnsJson + *******************************************************************************/ + public String getColumnsJson() + { + return (this.columnsJson); + } + + + + /******************************************************************************* + ** Setter for columnsJson + *******************************************************************************/ + public void setColumnsJson(String columnsJson) + { + this.columnsJson = columnsJson; + } + + + + /******************************************************************************* + ** Fluent setter for columnsJson + *******************************************************************************/ + public SavedReport withColumnsJson(String columnsJson) + { + this.columnsJson = columnsJson; + return (this); + } + + + + /******************************************************************************* + ** Getter for inputFieldsJson + *******************************************************************************/ + public String getInputFieldsJson() + { + return (this.inputFieldsJson); + } + + + + /******************************************************************************* + ** Setter for inputFieldsJson + *******************************************************************************/ + public void setInputFieldsJson(String inputFieldsJson) + { + this.inputFieldsJson = inputFieldsJson; + } + + + + /******************************************************************************* + ** Fluent setter for inputFieldsJson + *******************************************************************************/ + public SavedReport withInputFieldsJson(String inputFieldsJson) + { + this.inputFieldsJson = inputFieldsJson; + return (this); + } + + + + /******************************************************************************* + ** Getter for pivotTableJson + *******************************************************************************/ + public String getPivotTableJson() + { + return (this.pivotTableJson); + } + + + + /******************************************************************************* + ** Setter for pivotTableJson + *******************************************************************************/ + public void setPivotTableJson(String pivotTableJson) + { + this.pivotTableJson = pivotTableJson; + } + + + + /******************************************************************************* + ** Fluent setter for pivotTableJson + *******************************************************************************/ + public SavedReport withPivotTableJson(String pivotTableJson) + { + this.pivotTableJson = pivotTableJson; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatter.java new file mode 100644 index 00000000..cb1af39c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatter.java @@ -0,0 +1,150 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.SavedReportToReportMetaDataAdapter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedReportJsonFieldDisplayValueFormatter implements FieldDisplayBehavior +{ + private static SavedReportJsonFieldDisplayValueFormatter savedReportJsonFieldDisplayValueFormatter = null; + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private SavedReportJsonFieldDisplayValueFormatter() + { + + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static SavedReportJsonFieldDisplayValueFormatter getInstance() + { + if(savedReportJsonFieldDisplayValueFormatter == null) + { + savedReportJsonFieldDisplayValueFormatter = new SavedReportJsonFieldDisplayValueFormatter(); + } + return (savedReportJsonFieldDisplayValueFormatter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SavedReportJsonFieldDisplayValueFormatter getDefault() + { + return getInstance(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + if(field.getName().equals("queryFilterJson")) + { + String queryFilterJson = record.getValueString("queryFilterJson"); + if(StringUtils.hasContent(queryFilterJson)) + { + try + { + QQueryFilter qQueryFilter = SavedReportToReportMetaDataAdapter.getQQueryFilter(queryFilterJson); + int criteriaCount = CollectionUtils.nonNullList(qQueryFilter.getCriteria()).size(); + record.setDisplayValue("queryFilterJson", criteriaCount + " Filter" + StringUtils.plural(criteriaCount)); + } + catch(Exception e) + { + record.setDisplayValue("queryFilterJson", "Invalid Filter..."); + } + } + } + + if(field.getName().equals("columnsJson")) + { + String columnsJson = record.getValueString("columnsJson"); + if(StringUtils.hasContent(columnsJson)) + { + try + { + ReportColumns reportColumns = SavedReportToReportMetaDataAdapter.getReportColumns(columnsJson); + int columnCount = reportColumns.extractVisibleColumns().size(); + + record.setDisplayValue("columnsJson", columnCount + " Column" + StringUtils.plural(columnCount)); + } + catch(Exception e) + { + record.setDisplayValue("columnsJson", "Invalid Columns..."); + } + } + } + + if(field.getName().equals("pivotTableJson")) + { + String pivotTableJson = record.getValueString("pivotTableJson"); + if(StringUtils.hasContent(pivotTableJson)) + { + try + { + PivotTableDefinition pivotTableDefinition = SavedReportToReportMetaDataAdapter.getPivotTableDefinition(pivotTableJson); + int rowCount = CollectionUtils.nonNullList(pivotTableDefinition.getRows()).size(); + int columnCount = CollectionUtils.nonNullList(pivotTableDefinition.getColumns()).size(); + int valueCount = CollectionUtils.nonNullList(pivotTableDefinition.getValues()).size(); + record.setDisplayValue("pivotTableJson", rowCount + " Row" + StringUtils.plural(rowCount) + ", " + columnCount + " Column" + StringUtils.plural(columnCount) + ", and " + valueCount + " Value" + StringUtils.plural(valueCount)); + } + catch(Exception e) + { + record.setDisplayValue("pivotTableJson", "Invalid Pivot Table..."); + } + } + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java new file mode 100644 index 00000000..6fe6e9bf --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java @@ -0,0 +1,323 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.SavedReportToReportMetaDataAdapter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedReportTableCustomizer implements TableCustomizerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + return (preInsertOrUpdate(records)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + validateOwner(records, SavedReport.TABLE_NAME, "edit"); + return (preInsertOrUpdate(records)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preDelete(DeleteInput deleteInput, List records, boolean isPreview) throws QException + { + validateOwner(records, SavedReport.TABLE_NAME, "delete"); + return (preInsertOrUpdate(records)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void validateOwner(List records, String tableName, String verb) + { + QTableMetaData tableMetaData = QContext.getQInstance().getTable(tableName); + String currentUserId = ObjectUtils.tryElse(() -> QContext.getQSession().getUser().getIdReference(), null); + for(QRecord record : records) + { + if(record.getValue("userId") != null) + { + if(!record.getValue("userId").equals(currentUserId)) + { + record.addError(new PermissionDeniedMessage("Only the owner of a " + tableMetaData.getLabel() + " may " + verb + " it.")); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List preInsertOrUpdate(List records) + { + for(QRecord record : CollectionUtils.nonNullList(records)) + { + preValidateRecord(record); + } + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + void preValidateRecord(QRecord record) + { + try + { + String tableName = record.getValueString("tableName"); + String queryFilterJson = record.getValueString("queryFilterJson"); + String columnsJson = record.getValueString("columnsJson"); + String pivotTableJson = record.getValueString("pivotTableJson"); + + Set usedColumns = new HashSet<>(); + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table == null) + { + record.addError(new BadInputStatusMessage("Unrecognized table name: " + tableName)); + } + + if(StringUtils.hasContent(queryFilterJson)) + { + try + { + ///////////////////////////////////////////////////////////////////////// + // validate that we can parse the filter, then prep it for the backend // + ///////////////////////////////////////////////////////////////////////// + QQueryFilter filter = SavedReportToReportMetaDataAdapter.getQQueryFilter(queryFilterJson); + filter.prepForBackend(); + record.setValue("queryFilterJson", JsonUtils.toJson(filter)); + } + catch(IOException e) + { + record.addError(new BadInputStatusMessage("Unable to parse queryFilterJson: " + e.getMessage())); + } + } + + boolean hadColumnParseError = false; + if(StringUtils.hasContent(columnsJson)) + { + try + { + ///////////////////////////////////////////////////////////////////////// + // make sure we can parse columns, and that we have at least 1 visible // + ///////////////////////////////////////////////////////////////////////// + ReportColumns reportColumns = SavedReportToReportMetaDataAdapter.getReportColumns(columnsJson); + for(ReportColumn column : reportColumns.extractVisibleColumns()) + { + usedColumns.add(column.getName()); + } + } + catch(IOException e) + { + record.addError(new BadInputStatusMessage("Unable to parse columnsJson: " + e.getMessage())); + hadColumnParseError = true; + } + } + + if(usedColumns.isEmpty() && !hadColumnParseError) + { + record.addError(new BadInputStatusMessage("A Report must contain at least 1 column")); + } + + if(StringUtils.hasContent(pivotTableJson)) + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure we can parse pivot table, and we have ... at least 1 ... row? maybe that's all that's needed // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + PivotTableDefinition pivotTableDefinition = SavedReportToReportMetaDataAdapter.getPivotTableDefinition(pivotTableJson); + boolean anyRows = false; + boolean missingAnyFieldNamesInRows = false; + boolean missingAnyFieldNamesInColumns = false; + boolean missingAnyFieldNamesInValues = false; + boolean missingAnyFunctionsInValues = false; + + ////////////////// + // look at rows // + ////////////////// + for(PivotTableGroupBy row : CollectionUtils.nonNullList(pivotTableDefinition.getRows())) + { + anyRows = true; + if(StringUtils.hasContent(row.getFieldName())) + { + if(!usedColumns.contains(row.getFieldName()) && !hadColumnParseError) + { + record.addError(new BadInputStatusMessage("A pivot table row is using field (" + getFieldLabelElseName(table, row.getFieldName()) + ") which is not an active column on this report.")); + } + } + else + { + missingAnyFieldNamesInRows = true; + } + } + + if(!anyRows) + { + record.addError(new BadInputStatusMessage("A Pivot Table must contain at least 1 row")); + } + + ///////////////////// + // look at columns // + ///////////////////// + for(PivotTableGroupBy column : CollectionUtils.nonNullList(pivotTableDefinition.getColumns())) + { + if(StringUtils.hasContent(column.getFieldName())) + { + if(!usedColumns.contains(column.getFieldName()) && !hadColumnParseError) + { + record.addError(new BadInputStatusMessage("A pivot table column is using field (" + getFieldLabelElseName(table, column.getFieldName()) + ") which is not an active column on this report.")); + } + } + else + { + missingAnyFieldNamesInColumns = true; + } + } + + //////////////////// + // look at values // + //////////////////// + for(PivotTableValue value : CollectionUtils.nonNullList(pivotTableDefinition.getValues())) + { + if(StringUtils.hasContent(value.getFieldName())) + { + if(!usedColumns.contains(value.getFieldName()) && !hadColumnParseError) + { + record.addError(new BadInputStatusMessage("A pivot table value is using field (" + getFieldLabelElseName(table, value.getFieldName()) + ") which is not an active column on this report.")); + } + } + else + { + missingAnyFieldNamesInValues = true; + } + + if(value.getFunction() == null) + { + missingAnyFunctionsInValues = true; + } + } + + //////////////////////////////////////////////// + // errors based on missing things found above // + //////////////////////////////////////////////// + if(missingAnyFieldNamesInRows) + { + record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table row.")); + } + + if(missingAnyFieldNamesInColumns) + { + record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table column.")); + } + + if(missingAnyFieldNamesInValues) + { + record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table value.")); + } + + if(missingAnyFunctionsInValues) + { + record.addError(new BadInputStatusMessage("Missing function for at least one pivot table value.")); + } + } + catch(IOException e) + { + record.addError(new BadInputStatusMessage("Unable to parse pivotTableJson: " + e.getMessage())); + } + } + } + catch(Exception e) + { + LOG.warn("Error validating a savedReport"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFieldLabelElseName(QTableMetaData table, String fieldName) + { + try + { + GenerateReportAction.FieldAndJoinTable fieldAndJoinTable = GenerateReportAction.getFieldAndJoinTable(table, fieldName); + return (fieldAndJoinTable.getLabel(table)); + } + catch(Exception e) + { + return (fieldName); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsFilterAndColumnsSetupRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsFilterAndColumnsSetupRenderer.java new file mode 100644 index 00000000..d7b359f3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsFilterAndColumnsSetupRenderer.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.FilterAndColumnsSetupData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedReportsFilterAndColumnsSetupRenderer extends AbstractWidgetRenderer +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + return (new RenderWidgetOutput(new FilterAndColumnsSetupData(null, true, false, null))); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java new file mode 100644 index 00000000..2b40bd86 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java @@ -0,0 +1,406 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DefaultWidgetRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.common.TimeZonePossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RunScheduledReportMetaDataProducer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedReportsMetaDataProvider +{ + public static final String REPORT_STORAGE_TABLE_NAME = "reportStorage"; + + public static final String SAVED_REPORT_JOIN_SCHEDULED_REPORT = "scheduledReportJoinSavedReport"; + public static final String SHARED_SAVED_REPORT_JOIN_SAVED_REPORT = "sharedSavedReportJoinSavedReport"; + + public static final String SCHEDULED_REPORT_VALUES_WIDGET = "scheduledReportValuesWidget"; + public static final String RENDER_REPORT_PROCESS_VALUES_WIDGET = "renderReportProcessValuesWidget"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String recordTablesBackendName, String reportStorageBackendName, Consumer backendDetailEnricher) throws QException + { + instance.addTable(defineSavedReportTable(recordTablesBackendName, backendDetailEnricher)); + instance.addTable(defineRenderedReportTable(recordTablesBackendName, backendDetailEnricher)); + + instance.addPossibleValueSource(QPossibleValueSource.newForTable(SavedReport.TABLE_NAME)); + instance.addPossibleValueSource(QPossibleValueSource.newForEnum(ReportFormatPossibleValueEnum.NAME, ReportFormatPossibleValueEnum.values())); + instance.addPossibleValueSource(QPossibleValueSource.newForEnum(RenderedReportStatus.NAME, RenderedReportStatus.values())); + + instance.addTable(defineReportStorageTable(reportStorageBackendName, backendDetailEnricher)); + + QProcessMetaData renderSavedReportProcess = new RenderSavedReportMetaDataProducer().produce(instance); + instance.addProcess(renderSavedReportProcess); + renderSavedReportProcess.getInputFields().stream() + .filter(f -> RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME.equals(f.getName())) + .findFirst() + .ifPresent(f -> f.setDefaultValue(REPORT_STORAGE_TABLE_NAME)); + instance.addWidget(defineRenderReportProcessValuesWidget()); + + instance.addWidget(defineReportSetupWidget()); + instance.addWidget(definePivotTableSetupWidget()); + + //////////////////////////////////////// + // todo - param to enable scheduling? // + //////////////////////////////////////// + instance.addTable(defineScheduledReportTable(recordTablesBackendName, backendDetailEnricher)); + QJoinMetaData join = defineSavedReportJoinScheduledReport(); + instance.addJoin(join); + instance.addWidget(defineScheduledReportJoinSavedReportWidget(join)); + QProcessMetaData scheduledReportSyncToScheduledJobProcess = new ScheduledReportSyncToScheduledJobProcess().produce(instance); + instance.addProcess(scheduledReportSyncToScheduledJobProcess); + instance.addWidget(defineScheduledReportValuesWidget()); + + QProcessMetaData runScheduledReportProcess = new RunScheduledReportMetaDataProducer().produce(instance); + instance.addProcess(runScheduledReportProcess); + + if(instance.getPossibleValueSource(TimeZonePossibleValueSourceMetaDataProvider.NAME) == null) + { + instance.addPossibleValueSource(new TimeZonePossibleValueSourceMetaDataProvider().produce()); + } + + ///////////////////////////////////// + // todo - param to enable sharing? // + ///////////////////////////////////// + instance.addTable(defineSharedSavedReportTable(recordTablesBackendName, backendDetailEnricher)); + instance.addJoin(defineSharedSavedReportJoinSavedReport()); + if(instance.getPossibleValueSource(ShareScopePossibleValueMetaDataProducer.NAME) == null) + { + instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance())); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QWidgetMetaDataInterface defineScheduledReportValuesWidget() + { + return new QWidgetMetaData() + .withName(SCHEDULED_REPORT_VALUES_WIDGET) + .withType(WidgetType.DYNAMIC_FORM.getType()) + .withIsCard(true) + .withLabel("Variable Values") + .withCodeReference(new QCodeReference(ReportValuesDynamicFormWidgetRenderer.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QWidgetMetaDataInterface defineRenderReportProcessValuesWidget() + { + return new QWidgetMetaData() + .withName(RENDER_REPORT_PROCESS_VALUES_WIDGET) + .withType(WidgetType.DYNAMIC_FORM.getType()) + .withIsCard(false) + .withDefaultValue("isEditable", true) + .withCodeReference(new QCodeReference(ReportValuesDynamicFormWidgetRenderer.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QJoinMetaData defineSavedReportJoinScheduledReport() + { + return (new QJoinMetaData() + .withName(SAVED_REPORT_JOIN_SCHEDULED_REPORT) + .withLeftTable(SavedReport.TABLE_NAME) + .withRightTable(ScheduledReport.TABLE_NAME) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "savedReportId"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QWidgetMetaDataInterface defineScheduledReportJoinSavedReportWidget(QJoinMetaData join) + { + return ChildRecordListRenderer.widgetMetaDataBuilder(join) + .withLabel("Schedules") + .withCanAddChildRecord(true) + .getWidgetMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QJoinMetaData defineSharedSavedReportJoinSavedReport() + { + return (new QJoinMetaData() + .withName(SHARED_SAVED_REPORT_JOIN_SAVED_REPORT) + .withLeftTable(SharedSavedReport.TABLE_NAME) + .withRightTable(SavedReport.TABLE_NAME) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("savedReportId", "id"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineReportStorageTable(String backendName, Consumer backendDetailEnricher) + { + QTableMetaData table = new QTableMetaData() + .withName(REPORT_STORAGE_TABLE_NAME) + .withBackendName(backendName) + .withPrimaryKeyField("reference") + .withField(new QFieldMetaData("reference", QFieldType.STRING)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB)); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QWidgetMetaDataInterface defineReportSetupWidget() + { + return new QWidgetMetaData() + .withName("reportSetupWidget") + .withLabel("Filters and Columns") + .withIsCard(true) + .withType(WidgetType.FILTER_AND_COLUMNS_SETUP.getType()) + .withCodeReference(new QCodeReference(SavedReportsFilterAndColumnsSetupRenderer.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QWidgetMetaDataInterface definePivotTableSetupWidget() + { + return new QWidgetMetaData() + .withName("pivotTableSetupWidget") + .withLabel("Pivot Table") + .withIsCard(true) + .withType(WidgetType.PIVOT_TABLE_SETUP.getType()) + .withCodeReference(new QCodeReference(DefaultWidgetRenderer.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSavedReportTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SavedReport.TABLE_NAME) + .withLabel("Report") + .withIcon(new QIcon().withName("article")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SavedReport.class) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) + .withSection(new QFieldSection("filtersAndColumns", new QIcon().withName("table_chart"), Tier.T2).withLabel("Filters and Columns").withWidgetName("reportSetupWidget")) + .withSection(new QFieldSection("pivotTable", new QIcon().withName("pivot_table_chart"), Tier.T2).withLabel("Pivot Table").withWidgetName("pivotTableSetupWidget")) + .withSection(new QFieldSection("schedule", new QIcon().withName("schedule"), Tier.T2).withWidgetName(SAVED_REPORT_JOIN_SCHEDULED_REPORT)) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("queryFilterJson", "columnsJson", "pivotTableJson")).withIsHidden(true)) + .withSection(new QFieldSection("hidden", new QIcon().withName("text_snippet"), Tier.T2, List.of("inputFieldsJson", "userId")).withIsHidden(true)) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + table.getField("queryFilterJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + table.getField("columnsJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + table.getField("pivotTableJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + + table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(SavedReportTableCustomizer.class)); + table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(SavedReportTableCustomizer.class)); + table.withCustomizer(TableCustomizers.PRE_DELETE_RECORD, new QCodeReference(SavedReportTableCustomizer.class)); + + table.withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(SharedSavedReport.TABLE_NAME) + .withAssetIdFieldName("savedReportId") + .withScopeFieldName("scope") + .withThisTableOwnerIdFieldName("userId") + .withAudienceType(new ShareableAudienceType().withName("user").withFieldName("userId"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSharedSavedReportTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SharedSavedReport.TABLE_NAME) + .withLabel("Shared Report") + .withIcon(new QIcon().withName("share")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("savedReportId") + .withBackendName(backendName) + .withUniqueKey(new UniqueKey("savedReportId", "userId")) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SharedSavedReport.class) + // todo - security key + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId", "userId"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("scope"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineRenderedReportTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(RenderedReport.TABLE_NAME) + .withIcon(new QIcon().withName("print")) + .withRecordLabelFormat("%s - %s") + .withRecordLabelFields("savedReportId", "startTime") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(RenderedReport.class) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId", "renderedReportStatusId"))) + .withSection(new QFieldSection("input", new QIcon().withName("input"), Tier.T2, List.of("userId", "reportFormat"))) + .withSection(new QFieldSection("output", new QIcon().withName("output"), Tier.T2, List.of("jobUuid", "resultPath", "rowCount", "errorMessage", "startTime", "endTime"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) + .withoutCapabilities(Capability.allWriteCapabilities()); + + table.getField("renderedReportStatusId").setAdornments(List.of(new FieldAdornment(AdornmentType.CHIP) + .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.RUNNING.getId(), "pending", AdornmentType.ChipValues.COLOR_SECONDARY)) + .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.COMPLETE.getId(), "check", AdornmentType.ChipValues.COLOR_SUCCESS)) + .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.FAILED.getId(), "error", AdornmentType.ChipValues.COLOR_ERROR)))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineScheduledReportTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(ScheduledReport.TABLE_NAME) + .withIcon(new QIcon().withName("schedule_send")) + .withRecordLabelFormat("%s (Schedule %s)") + .withRecordLabelFields("savedReportId", "id") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(ScheduledReport.class) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId"))) + .withSection(new QFieldSection("settings", new QIcon().withName("settings"), Tier.T2, List.of("cronExpression", "cronTimeZoneId", "isActive", "format"))) + .withSection(new QFieldSection("email", new QIcon().withName("email"), Tier.T2, List.of("toAddresses", "subject"))) + .withSection(new QFieldSection("variableValues", new QIcon().withName("data_object"), Tier.T2).withWidgetName(SCHEDULED_REPORT_VALUES_WIDGET)) + .withSection(new QFieldSection("hidden", new QIcon().withName("visibility_off"), Tier.T2, List.of("inputValues", "userId")).withIsHidden(true)) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(ScheduledReportTableCustomizer.class)); + table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(ScheduledReportTableCustomizer.class)); + table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ScheduledReportTableCustomizer.class)); + table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ScheduledReportTableCustomizer.class)); + table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(ScheduledReportTableCustomizer.class)); + + return (table); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReport.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReport.java new file mode 100644 index 00000000..59677ce0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReport.java @@ -0,0 +1,478 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.common.TimeZonePossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** Entity bean for the scheduled report table + *******************************************************************************/ +public class ScheduledReport extends QRecordEntity +{ + public static final String TABLE_NAME = "scheduledReport"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, possibleValueSourceName = SavedReport.TABLE_NAME) + private Integer savedReportId; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, isRequired = true) + private String cronExpression; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = TimeZonePossibleValueSourceMetaDataProvider.NAME, isRequired = true) + private String cronTimeZoneId; + + @QField(isRequired = true, defaultValue = "true") + private Boolean isActive; + + @QField(isRequired = true) + private String toAddresses; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String subject; + + @QField(isRequired = true, maxLength = 20, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ReportFormatPossibleValueEnum.NAME) + private String format; + + @QField() + private String inputValues; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID, label = "Owner") + private String userId; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledReport() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledReport(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Setter for id + ** + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Getter for createDate + ** + *******************************************************************************/ + public Instant getCreateDate() + { + return createDate; + } + + + + /******************************************************************************* + ** Setter for createDate + ** + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Getter for modifyDate + ** + *******************************************************************************/ + public Instant getModifyDate() + { + return modifyDate; + } + + + + /******************************************************************************* + ** Setter for modifyDate + ** + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ScheduledReport withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ScheduledReport withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ScheduledReport withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for savedReportId + *******************************************************************************/ + public Integer getSavedReportId() + { + return (this.savedReportId); + } + + + + /******************************************************************************* + ** Setter for savedReportId + *******************************************************************************/ + public void setSavedReportId(Integer savedReportId) + { + this.savedReportId = savedReportId; + } + + + + /******************************************************************************* + ** Fluent setter for savedReportId + *******************************************************************************/ + public ScheduledReport withSavedReportId(Integer savedReportId) + { + this.savedReportId = savedReportId; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronExpression + *******************************************************************************/ + public String getCronExpression() + { + return (this.cronExpression); + } + + + + /******************************************************************************* + ** Setter for cronExpression + *******************************************************************************/ + public void setCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + } + + + + /******************************************************************************* + ** Fluent setter for cronExpression + *******************************************************************************/ + public ScheduledReport withCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronTimeZoneId + *******************************************************************************/ + public String getCronTimeZoneId() + { + return (this.cronTimeZoneId); + } + + + + /******************************************************************************* + ** Setter for cronTimeZoneId + *******************************************************************************/ + public void setCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + } + + + + /******************************************************************************* + ** Fluent setter for cronTimeZoneId + *******************************************************************************/ + public ScheduledReport withCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + return (this); + } + + + + /******************************************************************************* + ** Getter for isActive + *******************************************************************************/ + public Boolean getIsActive() + { + return (this.isActive); + } + + + + /******************************************************************************* + ** Setter for isActive + *******************************************************************************/ + public void setIsActive(Boolean isActive) + { + this.isActive = isActive; + } + + + + /******************************************************************************* + ** Fluent setter for isActive + *******************************************************************************/ + public ScheduledReport withIsActive(Boolean isActive) + { + this.isActive = isActive; + return (this); + } + + + + /******************************************************************************* + ** Getter for toAddresses + *******************************************************************************/ + public String getToAddresses() + { + return (this.toAddresses); + } + + + + /******************************************************************************* + ** Setter for toAddresses + *******************************************************************************/ + public void setToAddresses(String toAddresses) + { + this.toAddresses = toAddresses; + } + + + + /******************************************************************************* + ** Fluent setter for toAddresses + *******************************************************************************/ + public ScheduledReport withToAddresses(String toAddresses) + { + this.toAddresses = toAddresses; + return (this); + } + + + + /******************************************************************************* + ** Getter for subject + *******************************************************************************/ + public String getSubject() + { + return (this.subject); + } + + + + /******************************************************************************* + ** Setter for subject + *******************************************************************************/ + public void setSubject(String subject) + { + this.subject = subject; + } + + + + /******************************************************************************* + ** Fluent setter for subject + *******************************************************************************/ + public ScheduledReport withSubject(String subject) + { + this.subject = subject; + return (this); + } + + + + /******************************************************************************* + ** Getter for format + *******************************************************************************/ + public String getFormat() + { + return (this.format); + } + + + + /******************************************************************************* + ** Setter for format + *******************************************************************************/ + public void setFormat(String format) + { + this.format = format; + } + + + + /******************************************************************************* + ** Fluent setter for format + *******************************************************************************/ + public ScheduledReport withFormat(String format) + { + this.format = format; + return (this); + } + + + + /******************************************************************************* + ** Getter for inputValues + *******************************************************************************/ + public String getInputValues() + { + return (this.inputValues); + } + + + + /******************************************************************************* + ** Setter for inputValues + *******************************************************************************/ + public void setInputValues(String inputValues) + { + this.inputValues = inputValues; + } + + + + /******************************************************************************* + ** Fluent setter for inputValues + *******************************************************************************/ + public ScheduledReport withInputValues(String inputValues) + { + this.inputValues = inputValues; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public ScheduledReport withUserId(String userId) + { + this.userId = userId; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReportSyncToScheduledJobProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReportSyncToScheduledJobProcess.java new file mode 100644 index 00000000..68f0cbd3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReportSyncToScheduledJobProcess.java @@ -0,0 +1,189 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RunScheduledReportMetaDataProducer; +import com.kingsrook.qqq.backend.core.processes.implementations.tablesync.AbstractTableSyncTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.tablesync.TableSyncProcess; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledReportSyncToScheduledJobProcess extends AbstractTableSyncTransformStep implements MetaDataProducerInterface +{ + public static final String NAME = "scheduledReportSyncToScheduledJob"; + + public static final String SCHEDULER_NAME_FIELD_NAME = "schedulerName"; + + private static final QLogger LOG = QLogger.getLogger(ScheduledReportSyncToScheduledJobProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + QProcessMetaData processMetaData = TableSyncProcess.processMetaDataBuilder(false) + .withName(NAME) + .withTableName(ScheduledReport.TABLE_NAME) + + ///////////////////////////////////////// + // todo - maybe - to keep 'em in sync? // + ///////////////////////////////////////// + //.withBasepullConfiguration(CoreMetaDataProvider.getDefaultBasepullConfiguration("modifyDate", ONE_DAY_IN_HOURS) + // .withSecondsToSubtractFromLastRunTimeForTimestampQuery(10 * 60)) + // .withSchedule(new QScheduleMetaData() + // .withRepeatSeconds(SYNC_BASEPULLS_INTERVAL_SECONDS)) + + .withSyncTransformStepClass(getClass()) + .withReviewStepRecordFields(List.of( + new QFieldMetaData("savedReportId", QFieldType.INTEGER).withPossibleValueSourceName(SavedReport.TABLE_NAME), + new QFieldMetaData("cronExpression", QFieldType.STRING), + new QFieldMetaData("isActive", QFieldType.BOOLEAN), + new QFieldMetaData("toAddresses", QFieldType.STRING), + new QFieldMetaData("subject", QFieldType.STRING), + new QFieldMetaData("format", QFieldType.STRING).withPossibleValueSourceName(ReportFormatPossibleValueEnum.NAME) + )) + .getProcessMetaData(); + + processMetaData.getBackendStep(StreamedETLWithFrontendProcess.STEP_NAME_PREVIEW).getInputMetaData() + .withField(new QFieldMetaData(SCHEDULER_NAME_FIELD_NAME, QFieldType.STRING)); + + return (processMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException + { + ScheduledReport scheduledReport = new ScheduledReport(sourceRecord); + ScheduledJob scheduledJob; + + if(destinationRecord == null || destinationRecord.getValue("id") == null) + { + //////////////////////////////////////////////////////////////////////// + // need to do an insert - set lots of key values in the scheduled job // + //////////////////////////////////////////////////////////////////////// + scheduledJob = new ScheduledJob(); + scheduledJob.setLabel("Scheduled Report " + scheduledReport.getId()); + scheduledJob.setDescription("Job to run Scheduled Report Id " + scheduledReport.getId() + + " (which runs Report Id " + scheduledReport.getSavedReportId() + ")"); + scheduledJob.setSchedulerName(runBackendStepInput.getValueString(SCHEDULER_NAME_FIELD_NAME)); + scheduledJob.setType(ScheduledJobType.PROCESS.name()); + scheduledJob.setForeignKeyType(getScheduledJobForeignKeyType()); + scheduledJob.setForeignKeyValue(String.valueOf(scheduledReport.getId())); + scheduledJob.setJobParameters(List.of( + new ScheduledJobParameter().withKey("processName").withValue(getProcessNameScheduledJobParameter()), + new ScheduledJobParameter().withKey("recordId").withValue(ValueUtils.getValueAsString(scheduledReport.getId())) + )); + } + else + { + ////////////////////////////////////////////////////////////////////////////////// + // else doing an update - populate scheduled job entity from destination record // + ////////////////////////////////////////////////////////////////////////////////// + scheduledJob = new ScheduledJob(destinationRecord); + } + + ////////////////////////////////////////////////////////////////////////////////// + // these fields sync on insert and update // + // todo - if no diffs, should we return null (to avoid changing quartz at all?) // + ////////////////////////////////////////////////////////////////////////////////// + scheduledJob.setCronExpression(scheduledReport.getCronExpression()); + scheduledJob.setCronTimeZoneId(scheduledReport.getCronTimeZoneId()); + scheduledJob.setIsActive(scheduledReport.getIsActive()); + + return scheduledJob.toQRecord(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static String getScheduledJobForeignKeyType() + { + return "scheduledReport"; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String getProcessNameScheduledJobParameter() + { + return RunScheduledReportMetaDataProducer.NAME; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List sourceKeyList) + { + return super.getExistingRecordQueryFilter(runBackendStepInput, sourceKeyList) + .withCriteria(new QFilterCriteria("foreignKeyType", QCriteriaOperator.EQUALS, getScheduledJobForeignKeyType())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected SyncProcessConfig getSyncProcessConfig() + { + return new SyncProcessConfig(ScheduledReport.TABLE_NAME, "id", ScheduledJob.TABLE_NAME, "foreignKeyValue", true, true); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReportTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReportTableCustomizer.java new file mode 100644 index 00000000..b7d339f7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ScheduledReportTableCustomizer.java @@ -0,0 +1,217 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.Serializable; +import java.text.ParseException; +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValidationUtils; +import org.quartz.CronScheduleBuilder; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledReportTableCustomizer implements TableCustomizerInterface +{ + private static final QLogger LOG = QLogger.getLogger(ScheduledReportTableCustomizer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + preInsertOrUpdate(records); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + preInsertOrUpdate(records); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void preInsertOrUpdate(List records) + { + for(QRecord record : records) + { + String cronExpression = record.getValueString("cronExpression"); + try + { + CronScheduleBuilder.cronScheduleNonvalidatedExpression(cronExpression); + } + catch(ParseException e) + { + record.addError(new BadInputStatusMessage("Cron Expression [" + cronExpression + "] is not valid: " + e.getMessage())); + } + + try + { + String toAddresses = record.getValueString("toAddresses"); + if(StringUtils.hasContent(toAddresses)) + { + ValidationUtils.parseAndValidateEmailAddresses(toAddresses); + } + } + catch(QUserFacingException ufe) + { + record.addError(new BadInputStatusMessage(ufe.getMessage())); + } + catch(Exception e) + { + record.addError(new BadInputStatusMessage("To Addresses is not valid: " + e.getMessage())); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + runSyncProcess(records); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + runSyncProcess(records); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runSyncProcess(List records) + { + List scheduledReportIds = records.stream() + .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())) + .map(r -> r.getValue("id")).toList(); + + if(CollectionUtils.nullSafeIsEmpty(scheduledReportIds)) + { + return; + } + + try + { + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(ScheduledReportSyncToScheduledJobProcess.NAME); + runProcessInput.setCallback(QProcessCallbackFactory.forPrimaryKeys("id", scheduledReportIds)); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + + Serializable processSummary = runProcessOutput.getValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + System.out.println(processSummary); + } + catch(Exception e) + { + LOG.warn("Error syncing scheduled reports to scheduled jobs", e, logPair("scheduledReportIds", scheduledReportIds)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + List scheduledReportIds = records.stream() + .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())) + .map(r -> r.getValueString("id")).toList(); + + if(scheduledReportIds.isEmpty()) + { + return (records); + } + + /////////////////////////////////////////////////// + // delete any corresponding scheduledJob records // + /////////////////////////////////////////////////// + try + { + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(ScheduledJob.TABLE_NAME).withQueryFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("foreignKeyType", QCriteriaOperator.EQUALS, ScheduledReportSyncToScheduledJobProcess.getScheduledJobForeignKeyType())) + .withCriteria(new QFilterCriteria("foreignKeyValue", QCriteriaOperator.IN, scheduledReportIds)) + )); + + } + catch(Exception e) + { + LOG.warn("Error deleting scheduled jobs for scheduled reports", e); + } + + return (records); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SharedSavedReport.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SharedSavedReport.java new file mode 100644 index 00000000..37ac78a5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SharedSavedReport.java @@ -0,0 +1,267 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; + + +/******************************************************************************* + ** Entity bean for the shared saved report table + *******************************************************************************/ +public class SharedSavedReport extends QRecordEntity +{ + public static final String TABLE_NAME = "sharedSavedReport"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = SavedReport.TABLE_NAME, label = "Report") + private Integer savedReportId; + + @QField(label = "User") + private String userId; + + @QField(possibleValueSourceName = ShareScopePossibleValueMetaDataProducer.NAME) + private String scope; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedReport() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedReport(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public SharedSavedReport withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public SharedSavedReport withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public SharedSavedReport withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for savedReportId + *******************************************************************************/ + public Integer getSavedReportId() + { + return (this.savedReportId); + } + + + + /******************************************************************************* + ** Setter for savedReportId + *******************************************************************************/ + public void setSavedReportId(Integer savedReportId) + { + this.savedReportId = savedReportId; + } + + + + /******************************************************************************* + ** Fluent setter for savedReportId + *******************************************************************************/ + public SharedSavedReport withSavedReportId(Integer savedReportId) + { + this.savedReportId = savedReportId; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public SharedSavedReport withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for scope + *******************************************************************************/ + public String getScope() + { + return (this.scope); + } + + + + /******************************************************************************* + ** Setter for scope + *******************************************************************************/ + public void setScope(String scope) + { + this.scope = scope; + } + + + + /******************************************************************************* + ** Fluent setter for scope + *******************************************************************************/ + public SharedSavedReport withScope(String scope) + { + this.scope = scope; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewTableCustomizer.java new file mode 100644 index 00000000..c397197a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewTableCustomizer.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedviews; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportTableCustomizer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedViewTableCustomizer implements TableCustomizerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + SavedReportTableCustomizer.validateOwner(records, SavedView.TABLE_NAME, "edit"); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preDelete(DeleteInput deleteInput, List records, boolean isPreview) throws QException + { + SavedReportTableCustomizer.validateOwner(records, SavedView.TABLE_NAME, "delete"); + return (records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java index 2581e67d..b40c3edc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java @@ -24,17 +24,26 @@ package com.kingsrook.qqq.backend.core.model.savedviews; import java.util.List; import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.DeleteSavedViewProcess; import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.QuerySavedViewProcess; import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.StoreSavedViewProcess; @@ -45,6 +54,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.Store *******************************************************************************/ public class SavedViewsMetaDataProvider { + public static final String SHARED_SAVED_VIEW_JOIN_SAVED_VIEW = "sharedSavedViewJoinSavedView"; /******************************************************************************* @@ -57,6 +67,16 @@ public class SavedViewsMetaDataProvider instance.addProcess(QuerySavedViewProcess.getProcessMetaData()); instance.addProcess(StoreSavedViewProcess.getProcessMetaData()); instance.addProcess(DeleteSavedViewProcess.getProcessMetaData()); + + ///////////////////////////////////// + // todo - param to enable sharing? // + ///////////////////////////////////// + instance.addTable(defineSharedSavedViewTable(backendName, backendDetailEnricher)); + instance.addJoin(defineSharedSavedViewJoinSavedView()); + if(instance.getPossibleValueSource(ShareScopePossibleValueMetaDataProducer.NAME) == null) + { + instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance())); + } } @@ -68,7 +88,8 @@ public class SavedViewsMetaDataProvider { QTableMetaData table = new QTableMetaData() .withName(SavedView.TABLE_NAME) - .withLabel("Saved View") + .withLabel("View") + .withIcon(new QIcon().withName("table_view")) .withRecordLabelFormat("%s") .withRecordLabelFields("label") .withBackendName(backendName) @@ -80,6 +101,9 @@ public class SavedViewsMetaDataProvider table.getField("viewJson").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json"))); + table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(SavedViewTableCustomizer.class)); + table.withCustomizer(TableCustomizers.PRE_DELETE_RECORD, new QCodeReference(SavedViewTableCustomizer.class)); + if(backendDetailEnricher != null) { backendDetailEnricher.accept(table); @@ -103,4 +127,50 @@ public class SavedViewsMetaDataProvider .withOrderByField("label"); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSharedSavedViewTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SharedSavedView.TABLE_NAME) + .withLabel("Shared View") + .withIcon(new QIcon().withName("share")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("savedViewId") + .withBackendName(backendName) + .withUniqueKey(new UniqueKey("savedViewId", "userId")) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SharedSavedView.class) + // todo - security key + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedViewId", "userId"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("scope"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QJoinMetaData defineSharedSavedViewJoinSavedView() + { + return (new QJoinMetaData() + .withName(SHARED_SAVED_VIEW_JOIN_SAVED_VIEW) + .withLeftTable(SharedSavedView.TABLE_NAME) + .withRightTable(SavedView.TABLE_NAME) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("savedViewId", "id"))); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SharedSavedView.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SharedSavedView.java new file mode 100644 index 00000000..7062e233 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SharedSavedView.java @@ -0,0 +1,265 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedviews; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; + + +/******************************************************************************* + ** Entity bean for the shared saved view table + *******************************************************************************/ +public class SharedSavedView extends QRecordEntity +{ + public static final String TABLE_NAME = "sharedSavedView"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = SavedView.TABLE_NAME, label = "View") + private Integer savedViewId; + + @QField(label = "User") + private String userId; + + @QField(possibleValueSourceName = ShareScopePossibleValueMetaDataProducer.NAME) + private String scope; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedView() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedView(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for savedViewId + *******************************************************************************/ + public Integer getSavedViewId() + { + return (this.savedViewId); + } + + + + /******************************************************************************* + ** Setter for savedViewId + *******************************************************************************/ + public void setSavedViewId(Integer savedViewId) + { + this.savedViewId = savedViewId; + } + + + + /******************************************************************************* + ** Fluent setter for savedViewId + *******************************************************************************/ + public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withSavedViewId(Integer savedViewId) + { + this.savedViewId = savedViewId; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for scope + *******************************************************************************/ + public String getScope() + { + return (this.scope); + } + + + + /******************************************************************************* + ** Setter for scope + *******************************************************************************/ + public void setScope(String scope) + { + this.scope = scope; + } + + + + /******************************************************************************* + ** Fluent setter for scope + *******************************************************************************/ + public com.kingsrook.qqq.backend.core.model.savedviews.SharedSavedView withScope(String scope) + { + this.scope = scope; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java new file mode 100644 index 00000000..117f07ac --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java @@ -0,0 +1,566 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.common.TimeZonePossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.data.QAssociation; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QIgnore; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MutableMap; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJob extends QRecordEntity +{ + public static final String TABLE_NAME = "scheduledJob"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE) + private String label; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS) + private String description; + + @QField(isRequired = true, label = "Scheduler", maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = SchedulersPossibleValueSource.NAME) + private String schedulerName; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String cronExpression; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = TimeZonePossibleValueSourceMetaDataProvider.NAME) + private String cronTimeZoneId; + + @QField(displayFormat = DisplayFormat.COMMAS) + private Integer repeatSeconds; + + @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ScheduledJobTypePossibleValueSource.NAME) + private String type; + + @QField(isRequired = true) + private Boolean isActive; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String foreignKeyType; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String foreignKeyValue; + + @QAssociation(name = ScheduledJobParameter.TABLE_NAME) + private List jobParameters; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJob() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJob(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ScheduledJob withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ScheduledJob withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ScheduledJob withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public ScheduledJob withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for description + *******************************************************************************/ + public String getDescription() + { + return (this.description); + } + + + + /******************************************************************************* + ** Setter for description + *******************************************************************************/ + public void setDescription(String description) + { + this.description = description; + } + + + + /******************************************************************************* + ** Fluent setter for description + *******************************************************************************/ + public ScheduledJob withDescription(String description) + { + this.description = description; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronExpression + *******************************************************************************/ + public String getCronExpression() + { + return (this.cronExpression); + } + + + + /******************************************************************************* + ** Setter for cronExpression + *******************************************************************************/ + public void setCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + } + + + + /******************************************************************************* + ** Fluent setter for cronExpression + *******************************************************************************/ + public ScheduledJob withCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronTimeZoneId + *******************************************************************************/ + public String getCronTimeZoneId() + { + return (this.cronTimeZoneId); + } + + + + /******************************************************************************* + ** Setter for cronTimeZoneId + *******************************************************************************/ + public void setCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + } + + + + /******************************************************************************* + ** Fluent setter for cronTimeZoneId + *******************************************************************************/ + public ScheduledJob withCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + return (this); + } + + + + /******************************************************************************* + ** Getter for isActive + *******************************************************************************/ + public Boolean getIsActive() + { + return (this.isActive); + } + + + + /******************************************************************************* + ** Setter for isActive + *******************************************************************************/ + public void setIsActive(Boolean isActive) + { + this.isActive = isActive; + } + + + + /******************************************************************************* + ** Fluent setter for isActive + *******************************************************************************/ + public ScheduledJob withIsActive(Boolean isActive) + { + this.isActive = isActive; + return (this); + } + + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public ScheduledJob withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public String getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public ScheduledJob withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** Getter for jobParameters + *******************************************************************************/ + public List getJobParameters() + { + return (this.jobParameters); + } + + + + /******************************************************************************* + ** Getter for jobParameters - but a map of just the key=value pairs. + *******************************************************************************/ + @QIgnore + public Map getJobParametersMap() + { + if(CollectionUtils.nullSafeIsEmpty(this.jobParameters)) + { + return (new HashMap<>()); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // wrap in mutable map, just to avoid any immutable or other bs from toMap's default // + /////////////////////////////////////////////////////////////////////////////////////// + return new MutableMap<>(jobParameters.stream().collect(Collectors.toMap(ScheduledJobParameter::getKey, ScheduledJobParameter::getValue))); + } + + + + /******************************************************************************* + ** Setter for jobParameters + *******************************************************************************/ + public void setJobParameters(List jobParameters) + { + this.jobParameters = jobParameters; + } + + + + /******************************************************************************* + ** Fluent setter for jobParameters + *******************************************************************************/ + public ScheduledJob withJobParameters(List jobParameters) + { + this.jobParameters = jobParameters; + return (this); + } + + + + /******************************************************************************* + ** Getter for repeatSeconds + *******************************************************************************/ + public Integer getRepeatSeconds() + { + return (this.repeatSeconds); + } + + + + /******************************************************************************* + ** Setter for repeatSeconds + *******************************************************************************/ + public void setRepeatSeconds(Integer repeatSeconds) + { + this.repeatSeconds = repeatSeconds; + } + + + + /******************************************************************************* + ** Fluent setter for repeatSeconds + *******************************************************************************/ + public ScheduledJob withRepeatSeconds(Integer repeatSeconds) + { + this.repeatSeconds = repeatSeconds; + return (this); + } + + + + /******************************************************************************* + ** Getter for foreignKeyType + *******************************************************************************/ + public String getForeignKeyType() + { + return (this.foreignKeyType); + } + + + + /******************************************************************************* + ** Setter for foreignKeyType + *******************************************************************************/ + public void setForeignKeyType(String foreignKeyType) + { + this.foreignKeyType = foreignKeyType; + } + + + + /******************************************************************************* + ** Fluent setter for foreignKeyType + *******************************************************************************/ + public ScheduledJob withForeignKeyType(String foreignKeyType) + { + this.foreignKeyType = foreignKeyType; + return (this); + } + + + + /******************************************************************************* + ** Getter for foreignKeyValue + *******************************************************************************/ + public String getForeignKeyValue() + { + return (this.foreignKeyValue); + } + + + + /******************************************************************************* + ** Setter for foreignKeyValue + *******************************************************************************/ + public void setForeignKeyValue(String foreignKeyValue) + { + this.foreignKeyValue = foreignKeyValue; + } + + + + /******************************************************************************* + ** Fluent setter for foreignKeyValue + *******************************************************************************/ + public ScheduledJob withForeignKeyValue(String foreignKeyValue) + { + this.foreignKeyValue = foreignKeyValue; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java new file mode 100644 index 00000000..ccfa816e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java @@ -0,0 +1,266 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobParameter extends QRecordEntity +{ + public static final String TABLE_NAME = "scheduledJobParameter"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = ScheduledJob.TABLE_NAME, isRequired = true) + private Integer scheduledJobId; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, isRequired = true) + private String key; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String value; + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJobParameter() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJobParameter(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ScheduledJobParameter withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ScheduledJobParameter withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ScheduledJobParameter withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for scheduledJobId + *******************************************************************************/ + public Integer getScheduledJobId() + { + return (this.scheduledJobId); + } + + + + /******************************************************************************* + ** Setter for scheduledJobId + *******************************************************************************/ + public void setScheduledJobId(Integer scheduledJobId) + { + this.scheduledJobId = scheduledJobId; + } + + + + /******************************************************************************* + ** Fluent setter for scheduledJobId + *******************************************************************************/ + public ScheduledJobParameter withScheduledJobId(Integer scheduledJobId) + { + this.scheduledJobId = scheduledJobId; + return (this); + } + + + + /******************************************************************************* + ** Getter for key + *******************************************************************************/ + public String getKey() + { + return (this.key); + } + + + + /******************************************************************************* + ** Setter for key + *******************************************************************************/ + public void setKey(String key) + { + this.key = key; + } + + + + /******************************************************************************* + ** Fluent setter for key + *******************************************************************************/ + public ScheduledJobParameter withKey(String key) + { + this.key = key; + return (this); + } + + + + /******************************************************************************* + ** Getter for value + *******************************************************************************/ + public String getValue() + { + return (this.value); + } + + + + /******************************************************************************* + ** Setter for value + *******************************************************************************/ + public void setValue(String value) + { + this.value = value; + } + + + + /******************************************************************************* + ** Fluent setter for value + *******************************************************************************/ + public ScheduledJobParameter withValue(String value) + { + this.value = value; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java new file mode 100644 index 00000000..56114667 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java @@ -0,0 +1,37 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +/******************************************************************************* + ** enum of core schedulable types that QQQ schedule manager directly knows about. + ** + ** note though, that applications can define their own schedulable types, + ** by adding SchedulableType meta-data to the QInstance, and providing classes + ** that implement SchedulableRunner. + *******************************************************************************/ +public enum ScheduledJobType +{ + PROCESS, + QUEUE_PROCESSOR, + TABLE_AUTOMATIONS +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java new file mode 100644 index 00000000..43d26dd5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobTypePossibleValueSource implements QCustomPossibleValueProvider +{ + public static final String NAME = "scheduledJobType"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + SchedulableType schedulableType = QContext.getQInstance().getSchedulableType(String.valueOf(idValue)); + if(schedulableType != null) + { + return schedulableTypeToPossibleValue(schedulableType); + } + + return null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> rs = new ArrayList<>(); + for(SchedulableType schedulableType : CollectionUtils.nonNullMap(QContext.getQInstance().getSchedulableTypes()).values()) + { + rs.add(schedulableTypeToPossibleValue(schedulableType)); + } + return rs; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValue schedulableTypeToPossibleValue(SchedulableType schedulableType) + { + return new QPossibleValue<>(schedulableType.getName(), schedulableType.getName()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java new file mode 100644 index 00000000..d8c43e26 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java @@ -0,0 +1,249 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers.ScheduledJobParameterTableCustomizer; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers.ScheduledJobTableCustomizer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobsMetaDataProvider +{ + private static final String JOB_PARAMETER_JOIN_NAME = QJoinMetaData.makeInferredJoinName(ScheduledJob.TABLE_NAME, ScheduledJobParameter.TABLE_NAME); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String backendName, Consumer backendDetailEnricher) throws QException + { + defineStandardTables(instance, backendName, backendDetailEnricher); + instance.addPossibleValueSource(QPossibleValueSource.newForTable(ScheduledJob.TABLE_NAME)); + instance.addPossibleValueSource(defineScheduledJobTypePossibleValueSource()); + instance.addPossibleValueSource(defineSchedulersPossibleValueSource()); + defineStandardJoins(instance); + defineStandardWidgets(instance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardWidgets(QInstance instance) + { + QJoinMetaData join = instance.getJoin(JOB_PARAMETER_JOIN_NAME); + instance.addWidget(ChildRecordListRenderer.widgetMetaDataBuilder(join) + .withCanAddChildRecord(true) + .withManageAssociationName(ScheduledJobParameter.TABLE_NAME) + .withLabel("Parameters") + .getWidgetMetaData() + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardJoins(QInstance instance) + { + instance.addJoin(new QJoinMetaData() + .withType(JoinType.ONE_TO_MANY) + .withLeftTable(ScheduledJob.TABLE_NAME) + .withRightTable(ScheduledJobParameter.TABLE_NAME) + .withJoinOn(new JoinOn("id", "scheduledJobId")) + .withOrderBy(new QFilterOrderBy("id")) + .withInferredName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardTables(QInstance instance, String backendName, Consumer backendDetailEnricher) throws QException + { + for(QTableMetaData tableMetaData : defineStandardTables(backendName, backendDetailEnricher)) + { + instance.addTable(tableMetaData); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List defineStandardTables(String backendName, Consumer backendDetailEnricher) throws QException + { + List rs = new ArrayList<>(); + rs.add(enrich(backendDetailEnricher, defineScheduledJobTable(backendName))); + rs.add(enrich(backendDetailEnricher, defineScheduledJobParameterTable(backendName))); + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData enrich(Consumer backendDetailEnricher, QTableMetaData table) + { + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineStandardTable(String backendName, String name, Class fieldsFromEntity) throws QException + { + return new QTableMetaData() + .withName(name) + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(fieldsFromEntity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineScheduledJobTable(String backendName) throws QException + { + QTableMetaData tableMetaData = defineStandardTable(backendName, ScheduledJob.TABLE_NAME, ScheduledJob.class) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "description"))) + .withSection(new QFieldSection("schedule", new QIcon().withName("alarm"), Tier.T2, List.of("cronExpression", "cronTimeZoneId", "repeatSeconds"))) + .withSection(new QFieldSection("settings", new QIcon().withName("tune"), Tier.T2, List.of("type", "isActive", "schedulerName", "foreignKeyType", "foreignKeyValue"))) + .withSection(new QFieldSection("parameters", new QIcon().withName("list"), Tier.T2).withWidgetName(JOB_PARAMETER_JOIN_NAME)) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + QCodeReference customizerReference = new QCodeReference(ScheduledJobTableCustomizer.class); + tableMetaData.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_DELETE_RECORD, customizerReference); + + tableMetaData.withAssociation(new Association() + .withName(ScheduledJobParameter.TABLE_NAME) + .withAssociatedTableName(ScheduledJobParameter.TABLE_NAME) + .withJoinName(JOB_PARAMETER_JOIN_NAME)); + + tableMetaData.withExposedJoin(new ExposedJoin() + .withJoinTable(ScheduledJobParameter.TABLE_NAME) + .withJoinPath(List.of(JOB_PARAMETER_JOIN_NAME)) + .withLabel("Parameters")); + + return (tableMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineScheduledJobParameterTable(String backendName) throws QException + { + QTableMetaData tableMetaData = defineStandardTable(backendName, ScheduledJobParameter.TABLE_NAME, ScheduledJobParameter.class) + .withRecordLabelFormat("%s - %s") + .withRecordLabelFields("scheduledJobId", "key") + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "scheduledJobId", "key", "value"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + + QCodeReference customizerReference = new QCodeReference(ScheduledJobParameterTableCustomizer.class); + tableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_DELETE_RECORD, customizerReference); + + tableMetaData.withExposedJoin(new ExposedJoin() + .withJoinTable(ScheduledJob.TABLE_NAME) + .withJoinPath(List.of(JOB_PARAMETER_JOIN_NAME)) + .withLabel("Scheduled Job")); + + return (tableMetaData); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QPossibleValueSource defineScheduledJobTypePossibleValueSource() + { + return (new QPossibleValueSource() + .withName(ScheduledJobTypePossibleValueSource.NAME) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(ScheduledJobTypePossibleValueSource.class))); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private QPossibleValueSource defineSchedulersPossibleValueSource() + { + return (new QPossibleValueSource() + .withName(SchedulersPossibleValueSource.NAME) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(SchedulersPossibleValueSource.class))); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java new file mode 100644 index 00000000..e46c81c0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulersPossibleValueSource implements QCustomPossibleValueProvider +{ + public static final String NAME = "schedulers"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + QSchedulerMetaData scheduler = QContext.getQInstance().getScheduler(String.valueOf(idValue)); + if(scheduler != null) + { + return schedulerToPossibleValue(scheduler); + } + + return null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> rs = new ArrayList<>(); + for(QSchedulerMetaData scheduler : CollectionUtils.nonNullMap(QContext.getQInstance().getSchedulers()).values()) + { + if(scheduler.mayUseInScheduledJobsTable()) + { + rs.add(schedulerToPossibleValue(scheduler)); + } + } + return rs; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValue schedulerToPossibleValue(QSchedulerMetaData scheduler) + { + return new QPossibleValue<>(scheduler.getName(), scheduler.getName()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java new file mode 100644 index 00000000..bcd911f5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java @@ -0,0 +1,228 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobParameterTableCustomizer implements TableCustomizerInterface +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + /////////////////////////////////////////////////////////////////////////////////////// + // if we're in this insert as a result of an insert (or update) on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume that the // + // parent table's customizer will do what needed to be done. // + /////////////////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, Optional.empty()); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void bumpParentRecords(List records, Optional> oldRecordList) throws QException + { + try + { + /////////////////////////////////////////////////////////////////////////////////////////// + // (listing) hash up the records by scheduledJobId - we'll use this to have a set of the // + // job ids, and in case we need to add warnings to them later // + /////////////////////////////////////////////////////////////////////////////////////////// + ListingHash recordsByJobId = new ListingHash<>(); + for(QRecord record : records) + { + recordsByJobId.add(record.getValueInteger("scheduledJobId"), record); + } + + Set scheduledJobIds = new HashSet<>(recordsByJobId.keySet()); + + //////////////////////////////////////////////////////////////////////////////// + // if we have an old record list (e.g., is an edit), add any job ids that are // + // in those too, e.g., in case moving a param from one job to another... // + // note, we won't line these up for doing a proper warning on these... // + //////////////////////////////////////////////////////////////////////////////// + if(oldRecordList.isPresent()) + { + for(QRecord oldRecord : oldRecordList.get()) + { + scheduledJobIds.add(oldRecord.getValueInteger("scheduledJobId")); + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // update the modify date on the scheduled jobs - to get their post-actions to run, to reschedule // + //////////////////////////////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(ScheduledJob.TABLE_NAME); + updateInput.setRecords(scheduledJobIds.stream() + .map(id -> new QRecord().withValue("id", id).withValue("modifyDate", Instant.now())) + .toList()); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + + //////////////////////////////////////////////////////////////////////////////////////// + // look for warnings on those jobs - and propagate them to the params we just stored. // + //////////////////////////////////////////////////////////////////////////////////////// + for(QRecord updatedScheduledJob : updateOutput.getRecords()) + { + if(CollectionUtils.nullSafeHasContents(updatedScheduledJob.getWarnings())) + { + for(QRecord paramToWarn : CollectionUtils.nonNullList(recordsByJobId.get(updatedScheduledJob.getValueInteger("id")))) + { + paramToWarn.setWarnings(updatedScheduledJob.getWarnings()); + } + } + } + } + catch(Exception e) + { + LOG.warn("Error in scheduledJobParameter post-crud", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // if we're in this update as a result of an update on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume // + // that the parent table's customizer will do what needed to be done. // + ///////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, oldRecordList); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // if we're in this update as a result of an update on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume // + // that the parent table's customizer will do what needed to be done. // + ///////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, Optional.empty()); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean isThisAnActionDirectlyOnThisTable() + { + Optional firstActionInStack = QContext.getFirstActionInStack(); + if(firstActionInStack.isPresent()) + { + if(firstActionInStack.get() instanceof AbstractTableActionInput tableActionInput) + { + if(!ScheduledJobParameter.TABLE_NAME.equals(tableActionInput.getTableName())) + { + return (false); + } + } + else if(firstActionInStack.get() instanceof RunProcessInput runProcessInput) + { + String tableName = runProcessInput.getValueString("tableName"); + if(StringUtils.hasContent(tableName)) + { + if(!ScheduledJobParameter.TABLE_NAME.equals(tableName)) + { + return (false); + } + } + } + } + return (true); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java new file mode 100644 index 00000000..13cfc0ec --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java @@ -0,0 +1,323 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.text.ParseException; +import java.util.Collections; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.quartz.CronScheduleBuilder; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobTableCustomizer implements TableCustomizerInterface +{ + private QBackendTransaction transaction = null; + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + transaction = insertInput.getTransaction(); + validateConditionalFields(records, Collections.emptyMap()); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + transaction = insertInput.getTransaction(); + scheduleJobsForRecordList(records); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + transaction = updateInput.getTransaction(); + Map freshOldRecordsWithAssociationsMap = CollectionUtils.recordsToMap(freshlyQueryForRecordsWithAssociations(oldRecordList.get()), "id", Integer.class); + + validateConditionalFields(records, freshOldRecordsWithAssociationsMap); + + if(isPreview || oldRecordList.isEmpty()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // refresh the old-records w/ versions that have associations - so we can use those in the post-update to property unschedule things // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ListIterator iterator = oldRecordList.get().listIterator(); + while(iterator.hasNext()) + { + QRecord record = iterator.next(); + QRecord freshRecord = freshOldRecordsWithAssociationsMap.get(record.getValue("id")); + if(freshRecord != null) + { + iterator.set(freshRecord); + } + } + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void validateConditionalFields(List records, Map freshOldRecordsWithAssociationsMap) + { + QRecord blankRecord = new QRecord(); + for(QRecord record : records) + { + QRecord oldRecord = Objects.requireNonNullElse(freshOldRecordsWithAssociationsMap.get(record.getValueInteger("id")), blankRecord); + String cronExpression = record.getValues().containsKey("cronExpression") ? record.getValueString("cronExpression") : oldRecord.getValueString("cronExpression"); + String cronTimeZoneId = record.getValues().containsKey("cronTimeZoneId") ? record.getValueString("cronTimeZoneId") : oldRecord.getValueString("cronTimeZoneId"); + String repeatSeconds = record.getValues().containsKey("repeatSeconds") ? record.getValueString("repeatSeconds") : oldRecord.getValueString("repeatSeconds"); + + if(StringUtils.hasContent(cronExpression)) + { + if(StringUtils.hasContent(repeatSeconds)) + { + record.addError(new BadInputStatusMessage("Cron Expression and Repeat Seconds may not both be given.")); + } + + try + { + CronScheduleBuilder.cronScheduleNonvalidatedExpression(cronExpression); + } + catch(ParseException e) + { + record.addError(new BadInputStatusMessage("Cron Expression [" + cronExpression + "] is not valid: " + e.getMessage())); + } + + if(!StringUtils.hasContent(cronTimeZoneId)) + { + record.addError(new BadInputStatusMessage("If a Cron Expression is given, then a Cron Time Zone is required.")); + } + } + else + { + if(!StringUtils.hasContent(repeatSeconds)) + { + record.addError(new BadInputStatusMessage("Either Cron Expression or Repeat Seconds must be given.")); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + transaction = updateInput.getTransaction(); + if(oldRecordList.isPresent()) + { + Set idsWithErrors = getRecordIdsWithErrors(records); + unscheduleJobsForRecordList(oldRecordList.get(), idsWithErrors); + } + + scheduleJobsForRecordList(records); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Set getRecordIdsWithErrors(List records) + { + return records.stream() + .filter(r -> !recordHasErrors().test(r)) + .map(r -> r.getValueInteger("id")) + .collect(Collectors.toSet()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + transaction = deleteInput.getTransaction(); + Set idsWithErrors = getRecordIdsWithErrors(records); + unscheduleJobsForRecordList(records, idsWithErrors); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void scheduleJobsForRecordList(List records) + { + List recordsWithoutErrors = records.stream().filter(recordHasErrors()).toList(); + if(CollectionUtils.nullSafeIsEmpty(recordsWithoutErrors)) + { + return; + } + + try + { + Map originalRecordMap = recordsWithoutErrors.stream().collect(Collectors.toMap(r -> r.getValueInteger("id"), r -> r)); + List freshRecordListWithAssociations = freshlyQueryForRecordsWithAssociations(recordsWithoutErrors); + + QScheduleManager scheduleManager = QScheduleManager.getInstance(); + for(QRecord record : freshRecordListWithAssociations) + { + try + { + scheduleManager.setupScheduledJob(new ScheduledJob(record)); + } + catch(Exception e) + { + LOG.warn("Caught exception while scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + if(originalRecordMap.containsKey(record.getValueInteger("id"))) + { + originalRecordMap.get(record.getValueInteger("id")).addWarning(new QWarningMessage("Error scheduling job: " + e.getMessage())); + } + } + } + } + catch(Exception e) + { + LOG.warn("Error scheduling jobs in post-action", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Predicate recordHasErrors() + { + return r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List freshlyQueryForRecordsWithAssociations(List records) throws QException + { + List idList = records.stream().map(r -> r.getValueInteger("id")).toList(); + + return new QueryAction().execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true) + .withTransaction(transaction) + .withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, idList)))) + .getRecords(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void unscheduleJobsForRecordList(List oldRecords, Set exceptIdsWithErrors) + { + try + { + QScheduleManager scheduleManager = QScheduleManager.getInstance(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for un-schedule - use the old records as they are - don't re-query them (they may not exist anymore!) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : oldRecords) + { + try + { + ScheduledJob scheduledJob = new ScheduledJob(record); + + if(exceptIdsWithErrors.contains(scheduledJob.getId())) + { + LOG.info("Will not unschedule the job for a record that had an error", logPair("id", scheduledJob.getId())); + continue; + } + + scheduleManager.unscheduleScheduledJob(scheduledJob); + } + catch(Exception e) + { + LOG.warn("Caught exception while un-scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + } + } + } + catch(Exception e) + { + LOG.warn("Error scheduling jobs in post-action", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java index ac2e313a..cf050a18 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java @@ -28,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -41,7 +42,7 @@ import com.kingsrook.qqq.backend.core.utils.collections.MutableMap; /******************************************************************************* ** *******************************************************************************/ -public class QSession implements Serializable +public class QSession implements Serializable, Cloneable { private String idReference; private QUser user; @@ -52,14 +53,73 @@ public class QSession implements Serializable private Map> securityKeyValues; private Map backendVariants; - // implementation-specific custom values + /////////////////////////////////////////// + // implementation-specific custom values // + /////////////////////////////////////////// private Map values; + ///////////////////////////////////////////// + // values meant to be passed to a frontend // + ///////////////////////////////////////////// + private Map valuesForFrontend; + public static final String VALUE_KEY_USER_TIMEZONE = "UserTimezone"; public static final String VALUE_KEY_USER_TIMEZONE_OFFSET_MINUTES = "UserTimezoneOffsetMinutes"; + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSession clone() throws CloneNotSupportedException + { + QSession clone = (QSession) super.clone(); + + if(user != null) + { + clone.user = user.clone(); + } + + if(permissions != null) + { + clone.permissions = new HashSet<>(); + clone.permissions.addAll(permissions); + } + + if(securityKeyValues != null) + { + clone.securityKeyValues = new HashMap<>(); + for(Map.Entry> entry : securityKeyValues.entrySet()) + { + List cloneValues = entry.getValue() == null ? null : new ArrayList<>(entry.getValue()); + clone.securityKeyValues.put(entry.getKey(), cloneValues); + } + } + + if(backendVariants != null) + { + clone.backendVariants = new HashMap<>(); + clone.backendVariants.putAll(backendVariants); + } + + if(values != null) + { + clone.values = new HashMap<>(); + clone.values.putAll(values); + } + + if(valuesForFrontend != null) + { + clone.valuesForFrontend = new HashMap<>(); + clone.valuesForFrontend.putAll(valuesForFrontend); + } + + return (clone); + } + + + /******************************************************************************* ** Default constructor, puts a uuid in the session ** @@ -500,4 +560,49 @@ public class QSession implements Serializable return (this); } + + /******************************************************************************* + ** Getter for valuesForFrontend + *******************************************************************************/ + public Map getValuesForFrontend() + { + return (this.valuesForFrontend); + } + + + + /******************************************************************************* + ** Setter for valuesForFrontend + *******************************************************************************/ + public void setValuesForFrontend(Map valuesForFrontend) + { + this.valuesForFrontend = valuesForFrontend; + } + + + + /******************************************************************************* + ** Fluent setter for valuesForFrontend + *******************************************************************************/ + public QSession withValuesForFrontend(Map valuesForFrontend) + { + this.valuesForFrontend = valuesForFrontend; + return (this); + } + + + /******************************************************************************* + ** Fluent setter for a single valuesForFrontend + *******************************************************************************/ + public QSession withValueForFrontend(String key, Serializable value) + { + if(this.valuesForFrontend == null) + { + this.valuesForFrontend = new LinkedHashMap<>(); + } + this.valuesForFrontend.put(key, value); + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QUser.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QUser.java index 1ff995a0..1adcc3a4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QUser.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QUser.java @@ -25,13 +25,24 @@ package com.kingsrook.qqq.backend.core.model.session; /******************************************************************************* ** *******************************************************************************/ -public class QUser +public class QUser implements Cloneable { private String idReference; private String fullName; + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QUser clone() throws CloneNotSupportedException + { + return (QUser) super.clone(); + } + + + /******************************************************************************* ** Getter for idReference ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/statusmessages/DuplicateKeyBadInputStatusMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/statusmessages/DuplicateKeyBadInputStatusMessage.java new file mode 100644 index 00000000..ba26a029 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/statusmessages/DuplicateKeyBadInputStatusMessage.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.statusmessages; + + +/******************************************************************************* + ** specialization of bad-input status message, specifically for the case of + ** a duplicated key (e.g., unique-key validation error) + *******************************************************************************/ +public class DuplicateKeyBadInputStatusMessage extends BadInputStatusMessage +{ + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public DuplicateKeyBadInputStatusMessage(String message) + { + super(message); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleCustomizerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleCustomizerInterface.java index fd77cd81..021b8d4c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleCustomizerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleCustomizerInterface.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.modules.authentication; import java.io.Serializable; import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -52,4 +53,24 @@ public interface QAuthenticationModuleCustomizerInterface ////////// } + /******************************************************************************* + ** + *******************************************************************************/ + default void finalCustomizeSession(QInstance qInstance, QSession qSession) + { + ////////// + // noop // + ////////// + } + + /******************************************************************************* + ** + *******************************************************************************/ + default void customizeAutomatedSessionForUser(QInstance qInstance, QSession automatedSessionForUser, Serializable userId) throws QAuthenticationException + { + ////////// + // noop // + ////////// + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java index 6cd21ab6..28db59d6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java @@ -22,12 +22,15 @@ package com.kingsrook.qqq.backend.core.modules.authentication; +import java.io.Serializable; import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.AccessTokenException; import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang.NotImplementedException; @@ -49,6 +52,27 @@ public interface QAuthenticationModuleInterface boolean isSessionValid(QInstance instance, QSession session); + /******************************************************************************* + ** + *******************************************************************************/ + default QSession createAutomatedSessionForUser(QInstance qInstance, Serializable userId) throws QAuthenticationException + { + try + { + QSession clone = QContext.getQSession().clone(); + if(clone.getUser() != null) + { + clone.getUser().setIdReference(ValueUtils.getValueAsString(userId)); + } + return clone; + } + catch(CloneNotSupportedException e) + { + throw (new QAuthenticationException("Cloning session failed", e)); + } + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java index 7d1fc57d..91210020 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.modules.authentication.implementations; +import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.security.interfaces.RSAPublicKey; import java.time.Duration; @@ -149,10 +150,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface //////////////////////////////////////////////////////////////////////////////////////////////////////////// // this is how we allow the actions within this class to work without themselves having a logged-in user. // //////////////////////////////////////////////////////////////////////////////////////////////////////////// - private static QSession chickenAndEggSession = new QSession() - { - - }; + private static QSession chickenAndEggSession = null; @@ -162,14 +160,29 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface *******************************************************************************/ private QSession getChickenAndEggSession() { - for(String typeName : QContext.getQInstance().getSecurityKeyTypes().keySet()) + if(chickenAndEggSession == null) { - QSecurityKeyType keyType = QContext.getQInstance().getSecurityKeyType(typeName); - if(StringUtils.hasContent(keyType.getAllAccessKeyName())) + //////////////////////////////////////////////////////////////////////////////// + // if the static field is null, then let's make a new session; // + // prime it with all all-access keys; and then set it in the static field. // + // and, if 2 threads get in here at the same time, no real harm will be done, // + // other than creating the session twice, and whoever loses the race, that'll // + // be the one that stays in the field // + //////////////////////////////////////////////////////////////////////////////// + QSession newChickenAndEggSession = new QSession(); + + for(String typeName : QContext.getQInstance().getSecurityKeyTypes().keySet()) { - chickenAndEggSession = chickenAndEggSession.withSecurityKeyValue(keyType.getAllAccessKeyName(), true); + QSecurityKeyType keyType = QContext.getQInstance().getSecurityKeyType(typeName); + if(StringUtils.hasContent(keyType.getAllAccessKeyName())) + { + newChickenAndEggSession.withSecurityKeyValue(keyType.getAllAccessKeyName(), true); + } } + + chickenAndEggSession = newChickenAndEggSession; } + return (chickenAndEggSession); } @@ -195,7 +208,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // process a sessionUUID - looks up userSession record - cannot create token this way. // ///////////////////////////////////////////////////////////////////////////////////////// String sessionUUID = context.get(SESSION_UUID_KEY); - LOG.debug("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID))); + LOG.trace("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID))); if(sessionUUID != null) { accessToken = getAccessTokenFromSessionUUID(metaData, sessionUUID); @@ -230,6 +243,14 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface } } + ////////////////////////////////////////////////////////////// + // allow customizer to do custom things here, if so desired // + ////////////////////////////////////////////////////////////// + if(getCustomizer() != null) + { + getCustomizer().finalCustomizeSession(qInstance, qSession); + } + return (qSession); } else if(CollectionUtils.containsKeyWithNonNullValue(context, BASIC_AUTH_KEY)) @@ -245,7 +266,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // decode the credentials from the header auth // ///////////////////////////////////////////////// String base64Credentials = context.get(BASIC_AUTH_KEY).trim(); - LOG.info("Creating session from basicAuthentication", logPair("base64Credentials", maskForLog(base64Credentials))); + LOG.trace("Creating session from basicAuthentication", logPair("base64Credentials", maskForLog(base64Credentials))); accessToken = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials); } catch(Auth0Exception e) @@ -264,7 +285,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // process an api key - looks up client application token (creating token if needed) // /////////////////////////////////////////////////////////////////////////////////////// String apiKey = context.get(API_KEY); - LOG.info("Creating session from apiKey (accessTokenTable)", logPair("apiKey", maskForLog(apiKey))); + LOG.trace("Creating session from apiKey (accessTokenTable)", logPair("apiKey", maskForLog(apiKey))); if(apiKey != null) { accessToken = getAccessTokenFromApiKey(metaData, apiKey); @@ -284,7 +305,17 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // try to build session to see if still valid // // then call method to check more session validity // ///////////////////////////////////////////////////// - return buildAndValidateSession(qInstance, accessToken); + QSession qSession = buildAndValidateSession(qInstance, accessToken); + + ////////////////////////////////////////////////////////////// + // allow customizer to do custom things here, if so desired // + ////////////////////////////////////////////////////////////// + if(getCustomizer() != null) + { + getCustomizer().finalCustomizeSession(qInstance, qSession); + } + + return (qSession); } catch(QAuthenticationException qae) { @@ -604,7 +635,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // set security keys in the session from the JWT // /////////////////////////////////////////////////// setSecurityKeysInSessionFromJwtPayload(qInstance, payload, qSession); - + ////////////////////////////////////////////////////////////// // allow customizer to do custom things here, if so desired // ////////////////////////////////////////////////////////////// @@ -1090,4 +1121,20 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface return (null); } } + + + + /******************************************************************************* + ** e.g., if a scheduled job needs to run as a user (say, a report)... + *******************************************************************************/ + @Override + public QSession createAutomatedSessionForUser(QInstance qInstance, Serializable userId) throws QAuthenticationException + { + QSession automatedSessionForUser = QAuthenticationModuleInterface.super.createAutomatedSessionForUser(qInstance, userId); + if(getCustomizer() != null) + { + getCustomizer().customizeAutomatedSessionForUser(qInstance, automatedSessionForUser, userId); + } + return (automatedSessionForUser); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java index a4e8525e..d8069f0d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java @@ -33,14 +33,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; ** This class is responsible for loading a backend module, by its name, and ** returning an instance. ** - ** TODO - make this mapping runtime-bound, not pre-compiled in. - ** *******************************************************************************/ public class QBackendModuleDispatcher { private static final QLogger LOG = QLogger.getLogger(QBackendModuleDispatcher.class); - private static Map backendTypeToModuleClassNameMap; + private static Map backendTypeToModuleClassNameMap = new HashMap<>(); @@ -49,51 +47,6 @@ public class QBackendModuleDispatcher *******************************************************************************/ public QBackendModuleDispatcher() { - initBackendTypeToModuleClassNameMap(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static void initBackendTypeToModuleClassNameMap() - { - if(backendTypeToModuleClassNameMap != null) - { - return; - } - - Map newMap = new HashMap<>(); - - String[] moduleClassNames = new String[] - { - // todo - let modules somehow "export" their types here? - // e.g., backend-core shouldn't need to "know" about the modules. - "com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule", - "com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule", - "com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration.EnumerationBackendModule", - "com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule", - "com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule", - "com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule", - "com.kingsrook.qqq.backend.module.api.APIBackendModule" - }; - - for(String moduleClassName : moduleClassNames) - { - try - { - Class moduleClass = Class.forName(moduleClassName); - QBackendModuleInterface module = (QBackendModuleInterface) moduleClass.getConstructor().newInstance(); - newMap.put(module.getBackendType(), moduleClassName); - } - catch(Exception e) - { - LOG.debug("Backend module [{}] could not be loaded: {}", moduleClassName, e.getMessage()); - } - } - - backendTypeToModuleClassNameMap = newMap; } @@ -103,7 +56,6 @@ public class QBackendModuleDispatcher *******************************************************************************/ public static void registerBackendModule(QBackendModuleInterface moduleInstance) { - initBackendTypeToModuleClassNameMap(); String backendType = moduleInstance.getBackendType(); if(backendTypeToModuleClassNameMap.containsKey(backendType)) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java index 64ce0c3c..87eef4c5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java @@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -39,7 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails /******************************************************************************* ** Interface that a QBackendModule must implement. ** - ** Note, some methods all have a default version, which throws a 'not implemented' + ** Note, all methods have a default version, which throws a 'not implemented' ** exception. ** *******************************************************************************/ @@ -129,6 +130,16 @@ public interface QBackendModuleInterface return null; } + + /******************************************************************************* + ** + *******************************************************************************/ + default QStorageInterface getStorageInterface() + { + throwNotImplemented("StorageInterface"); + return null; + } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java index 7beb1ec2..53c1c594 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumerati import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -37,6 +38,10 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class EnumerationBackendModule implements QBackendModuleInterface { + static + { + QBackendModuleDispatcher.registerBackendModule(new EnumerationBackendModule()); + } /******************************************************************************* ** Method where a backend module must be able to provide its type (name). diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java index 4d6a93cb..3203280e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java @@ -26,8 +26,10 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -42,6 +44,12 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class MemoryBackendModule implements QBackendModuleInterface { + static + { + QBackendModuleDispatcher.registerBackendModule(new MemoryBackendModule()); + } + + /******************************************************************************* ** Method where a backend module must be able to provide its type (name). *******************************************************************************/ @@ -117,4 +125,14 @@ public class MemoryBackendModule implements QBackendModuleInterface return (new MemoryDeleteAction()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return (new MemoryStorageAction()); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 890d7c7f..dfd25fcf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -177,6 +177,14 @@ public class MemoryRecordStore for(QRecord qRecord : tableData) { + if(qRecord.getTableName() == null) + { + /////////////////////////////////////////////////////////////////////////////////////////// + // internally, doesRecordMatch likes to know table names on records, so, set if missing. // + /////////////////////////////////////////////////////////////////////////////////////////// + qRecord.setTableName(input.getTableName()); + } + boolean recordMatches = BackendQueryFilterUtils.doesRecordMatch(input.getFilter(), qRecord); if(recordMatches) @@ -232,16 +240,7 @@ public class MemoryRecordStore { QTableMetaData nextTable = qInstance.getTable(queryJoin.getJoinTable()); Collection nextTableRecords = getTableData(nextTable).values(); - - QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> - { - QJoinMetaData found = joinsContext.findJoinMetaData(qInstance, input.getTableName(), queryJoin.getJoinTable()); - if(found == null) - { - throw (new RuntimeException("Could not find a join between tables [" + input.getTableName() + "][" + queryJoin.getJoinTable() + "]")); - } - return (found); - }); + QJoinMetaData joinMetaData = Objects.requireNonNull(queryJoin.getJoinMetaData(), () -> "Could not find a join between tables [" + leftTable + "][" + queryJoin.getJoinTable() + "]"); List nextLevelProduct = new ArrayList<>(); for(QRecord productRecord : crossProduct) @@ -372,7 +371,7 @@ public class MemoryRecordStore ///////////////////////////////////////////////// // set the next serial in the record if needed // ///////////////////////////////////////////////// - if(recordToInsert.getValue(primaryKeyField.getName()) == null && primaryKeyField.getType().equals(QFieldType.INTEGER)) + if(recordToInsert.getValue(primaryKeyField.getName()) == null && (primaryKeyField.getType().equals(QFieldType.INTEGER) || primaryKeyField.getType().equals(QFieldType.LONG))) { recordToInsert.setValue(primaryKeyField.getName(), nextSerial++); } @@ -384,6 +383,13 @@ public class MemoryRecordStore { nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1; } + else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueLong(primaryKeyField.getName()) > nextSerial) + { + ////////////////////////////////////// + // todo - mmm, could overflow here? // + ////////////////////////////////////// + nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1; + } tableData.put(recordToInsert.getValue(primaryKeyField.getName()), recordToInsert); if(returnInsertedRecords) @@ -763,7 +769,6 @@ public class MemoryRecordStore /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") private static Serializable computeAggregate(List records, Aggregate aggregate, QTableMetaData table) { String fieldName = aggregate.getFieldName(); @@ -773,7 +778,7 @@ public class MemoryRecordStore { // todo - joins probably? QFieldMetaData field = table.getField(fieldName); - if(field.getType().equals(QFieldType.INTEGER) && (operator.equals(AggregateOperator.AVG))) + if((field.getType().equals(QFieldType.INTEGER) || field.getType().equals(QFieldType.LONG)) && (operator.equals(AggregateOperator.AVG))) { fieldType = QFieldType.DECIMAL; } @@ -809,6 +814,10 @@ public class MemoryRecordStore .filter(r -> r.getValue(fieldName) != null) .mapToInt(r -> r.getValueInteger(fieldName)) .sum(); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .sum(); case DECIMAL -> records.stream() .filter(r -> r.getValue(fieldName) != null) .map(r -> r.getValueBigDecimal(fieldName)) @@ -823,6 +832,11 @@ public class MemoryRecordStore .mapToInt(r -> r.getValueInteger(fieldName)) .min() .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .min() + .stream().boxed().findFirst().orElse(null); case DECIMAL, STRING, DATE, DATE_TIME -> { Optional serializable = records.stream() @@ -839,7 +853,12 @@ public class MemoryRecordStore { case INTEGER -> records.stream() .filter(r -> r.getValue(fieldName) != null) - .mapToInt(r -> r.getValueInteger(fieldName)) + .mapToLong(r -> r.getValueInteger(fieldName)) + .max() + .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) .max() .stream().boxed().findFirst().orElse(null); case DECIMAL, STRING, DATE, DATE_TIME -> @@ -861,6 +880,11 @@ public class MemoryRecordStore .mapToInt(r -> r.getValueInteger(fieldName)) .average() .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .average() + .stream().boxed().findFirst().orElse(null); case DECIMAL -> records.stream() .filter(r -> r.getValue(fieldName) != null) .mapToDouble(r -> r.getValueBigDecimal(fieldName).doubleValue()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryStorageAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryStorageAction.java new file mode 100644 index 00000000..313145a7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryStorageAction.java @@ -0,0 +1,149 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** implementation of bulk-storage interface, for the memory backend module. + ** + ** Requires table to have (at least?) 2 fields - a STRING primary key and a + ** BLOB to store bytes. + *******************************************************************************/ +public class MemoryStorageAction implements QStorageInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) + { + return new MemoryStorageOutputStream(storageInput.getTableName(), storageInput.getReference()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + QRecord record = new GetAction().executeForRecord(new GetInput(storageInput.getTableName()).withPrimaryKey(storageInput.getReference())); + if(record == null) + { + throw (new QNotFoundException("Could not find input stream for [" + storageInput.getTableName() + "][" + storageInput.getReference() + "]")); + } + + QFieldMetaData blobField = getBlobField(storageInput.getTableName()); + return (new ByteArrayInputStream(record.getValueByteArray(blobField.getName()))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QFieldMetaData getBlobField(String tableName) throws QException + { + Optional firstBlobField = QContext.getQInstance().getTable(tableName).getFields().values().stream().filter(f -> QFieldType.BLOB.equals(f.getType())).findFirst(); + if(firstBlobField.isEmpty()) + { + throw (new QException("Could not find a blob field in table [" + tableName + "]")); + } + return firstBlobField.get(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static class MemoryStorageOutputStream extends ByteArrayOutputStream + { + private final String tableName; + private final String reference; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public MemoryStorageOutputStream(String tableName, String reference) + { + this.tableName = tableName; + this.reference = reference; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void close() throws IOException + { + super.close(); + + try + { + QFieldMetaData blobField = getBlobField(tableName); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(tableName).withRecord(new QRecord() + .withValue(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), reference) + .withValue(blobField.getName(), toByteArray()))); + + if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors())) + { + throw(new IOException("Error storing stream into memory storage: " + StringUtils.joinWithCommasAndAnd(insertOutput.getRecords().get(0).getErrors().stream().map(e -> e.getMessage()).toList()))); + } + } + catch(Exception e) + { + throw new IOException("Wrapped QException", e); + } + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java index 9ee92353..e530e759 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java @@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -40,6 +41,11 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class MockBackendModule implements QBackendModuleInterface { + static + { + QBackendModuleDispatcher.registerBackendModule(new MockBackendModule()); + } + /******************************************************************************* ** Method where a backend module must be able to provide its type (name). *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockCountAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockCountAction.java index d64fba72..682acbfd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockCountAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockCountAction.java @@ -38,7 +38,6 @@ public class MockCountAction implements CountInterface /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:MagicNumber") public CountOutput execute(CountInput countInput) throws QException { try diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index 9fc87831..7a67a3b7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -95,14 +95,13 @@ public class MockQueryAction implements QueryInterface ** Get a mock value to use, based on its type. ** *******************************************************************************/ - @SuppressWarnings("checkstyle:MagicNumber") public static Serializable getMockValue(QTableMetaData table, String field) { - // @formatter:off // IJ can't do new-style switch correctly yet... return switch(table.getField(field).getType()) { case STRING -> UUID.randomUUID().toString(); case INTEGER -> 42; + case LONG -> 42L; case DECIMAL -> new BigDecimal("3.14159"); case DATE -> LocalDate.of(1970, Month.JANUARY, 1); case DATE_TIME -> LocalDateTime.of(1970, Month.JANUARY, 1, 0, 0); @@ -111,7 +110,6 @@ public class MockQueryAction implements QueryInterface case PASSWORD -> "abc***234"; default -> throw new IllegalStateException("Unexpected value: " + table.getField(field).getType()); }; - // @formatter:on } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java index 86fa2191..6862e01f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java @@ -32,6 +32,8 @@ import java.util.ListIterator; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -50,6 +52,9 @@ import org.apache.commons.lang.NotImplementedException; *******************************************************************************/ public class BackendQueryFilterUtils { + private static final QLogger LOG = QLogger.getLogger(BackendQueryFilterUtils.class); + + /******************************************************************************* ** Test if record matches filter. @@ -78,14 +83,16 @@ public class BackendQueryFilterUtils { /////////////////////////////////////////////////////////////////////////////////////////////////// // if the value isn't in the record - check, if it looks like a table.fieldName, but none of the // - // field names in the record are fully qualified, then just use the field-name portion... // + // field names in the record are fully qualified - OR - the table name portion of the field name // + // matches the record's field name, then just use the field-name portion... // /////////////////////////////////////////////////////////////////////////////////////////////////// if(fieldName.contains(".")) { + String[] parts = fieldName.split("\\."); Map values = qRecord.getValues(); - if(values.keySet().stream().noneMatch(n -> n.contains("."))) + if(values.keySet().stream().noneMatch(n -> n.contains(".")) || parts[0].equals(qRecord.getTableName())) { - value = qRecord.getValue(fieldName.substring(fieldName.indexOf(".") + 1)); + value = qRecord.getValue(parts[1]); } } } @@ -127,7 +134,6 @@ public class BackendQueryFilterUtils /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value) { ListIterator valueListIterator = criterion.getValues().listIterator(); @@ -136,7 +142,14 @@ public class BackendQueryFilterUtils Serializable criteriaValue = valueListIterator.next(); if(criteriaValue instanceof AbstractFilterExpression expression) { - valueListIterator.set(expression.evaluate()); + try + { + valueListIterator.set(expression.evaluate()); + } + catch(QException qe) + { + LOG.warn("Unexpected exception caught evaluating expression", qe); + } } } @@ -177,6 +190,8 @@ public class BackendQueryFilterUtils boolean between = (testGreaterThan(criteria0, value) || testEquals(criteria0, value)) && (!testGreaterThan(criteria1, value) || testEquals(criteria1, value)); yield !between; } + case TRUE -> true; + case FALSE -> false; }; return criterionMatches; } @@ -203,12 +218,13 @@ public class BackendQueryFilterUtils ** operator, update the accumulator, and if we can then short-circuit remaining ** operations, return a true or false. Returning null means to keep going. *******************************************************************************/ - private static Boolean applyBooleanOperator(AtomicBoolean accumulator, boolean newValue, QQueryFilter.BooleanOperator booleanOperator) + static Boolean applyBooleanOperator(AtomicBoolean accumulator, boolean newValue, QQueryFilter.BooleanOperator booleanOperator) { boolean accumulatorValue = accumulator.getPlain(); if(booleanOperator.equals(QQueryFilter.BooleanOperator.AND)) { accumulatorValue &= newValue; + accumulator.set(accumulatorValue); if(!accumulatorValue) { return (false); @@ -217,6 +233,7 @@ public class BackendQueryFilterUtils else { accumulatorValue |= newValue; + accumulator.set(accumulatorValue); if(accumulatorValue) { return (true); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/messaging/MessagingProviderInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/messaging/MessagingProviderInterface.java new file mode 100644 index 00000000..de7bdc74 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/messaging/MessagingProviderInterface.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.modules.messaging; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface MessagingProviderInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + String getType(); + + /******************************************************************************* + ** + *******************************************************************************/ + SendMessageOutput sendMessage(SendMessageInput sendMessageInput) throws QException; + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/messaging/QMessagingProviderDispatcher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/messaging/QMessagingProviderDispatcher.java new file mode 100644 index 00000000..4187fb39 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/messaging/QMessagingProviderDispatcher.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.modules.messaging; + + +import java.util.HashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData; + + +/******************************************************************************* + ** This class is responsible for loading a messaging provider, by its name, and + ** returning an instance. + ** + *******************************************************************************/ +public class QMessagingProviderDispatcher +{ + private static final QLogger LOG = QLogger.getLogger(QMessagingProviderDispatcher.class); + + private static Map typeToProviderClassNameMap; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QMessagingProviderDispatcher() + { + initBackendTypeToModuleClassNameMap(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void initBackendTypeToModuleClassNameMap() + { + if(typeToProviderClassNameMap != null) + { + return; + } + + Map newMap = new HashMap<>(); + + typeToProviderClassNameMap = newMap; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void registerMessagingProvider(MessagingProviderInterface messagingProviderInstance) + { + initBackendTypeToModuleClassNameMap(); + String type = messagingProviderInstance.getType(); + if(typeToProviderClassNameMap.containsKey(type)) + { + LOG.info("Overwriting messagingProvider type [" + type + "] with [" + messagingProviderInstance.getClass() + "]"); + } + typeToProviderClassNameMap.put(type, messagingProviderInstance.getClass().getName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MessagingProviderInterface getMessagingProviderInterface(QMessagingProviderMetaData messagingProviderMetaData) throws QModuleDispatchException + { + return (getMessagingProviderInterface(messagingProviderMetaData.getType())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MessagingProviderInterface getMessagingProviderInterface(String type) throws QModuleDispatchException + { + try + { + String className = typeToProviderClassNameMap.get(type); + if(className == null) + { + throw (new QModuleDispatchException("Unrecognized messaging provider type [" + type + "] in dispatcher.")); + } + + Class moduleClass = Class.forName(className); + return (MessagingProviderInterface) moduleClass.getDeclaredConstructor().newInstance(); + } + catch(QModuleDispatchException qmde) + { + throw (qmde); + } + catch(Exception e) + { + throw (new QModuleDispatchException("Error getting messaging provider of type: " + type, e)); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java index 84b3ab31..310fafe6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java @@ -55,6 +55,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; @@ -129,6 +130,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, QProcessMetaData processMetaData = new QProcessMetaData() .withName(NAME) + .withIcon(new QIcon().withName("healing")) .withStepList(List.of( new QFrontendStepMetaData() .withName("input") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java index 1c02a563..739619dd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProvi import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -71,6 +72,7 @@ public class RunTableAutomationsProcessStep implements BackendStep, MetaDataProd { QProcessMetaData processMetaData = new QProcessMetaData() .withName(NAME) + .withIcon(new QIcon().withName("directions_run")) .withStepList(List.of( new QFrontendStepMetaData() .withName("input") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java index 920da51f..3fdd208f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java @@ -112,12 +112,12 @@ public class BulkDeleteLoadStep extends LoadViaDeleteStep implements ProcessSumm ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { //////////////////////////// // have base class delete // //////////////////////////// - super.run(runBackendStepInput, runBackendStepOutput); + super.runOnePage(runBackendStepInput, runBackendStepOutput); QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); String primaryKeyFieldName = table.getPrimaryKeyField(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java index 0c016813..c0357f66 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java @@ -78,7 +78,7 @@ public class BulkDeleteTransformStep extends AbstractTransformStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { QTableMetaData table = runBackendStepInput.getTable(); String primaryKeyField = table.getPrimaryKeyField(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java index c05c2b64..e35181eb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java @@ -114,12 +114,12 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { //////////////////////////// // have base class update // //////////////////////////// - super.run(runBackendStepInput, runBackendStepOutput); + super.runOnePage(runBackendStepInput, runBackendStepOutput); //////////////////////////////////////////////////////// // roll up results based on output from update action // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java index 88d60eb6..624d6a04 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java @@ -103,7 +103,7 @@ public class BulkEditTransformStep extends AbstractTransformStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 7f33a624..2546df14 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -34,6 +34,7 @@ import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; @@ -117,7 +118,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { int rowsInThisPage = runBackendStepInput.getRecords().size(); QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); @@ -137,15 +138,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep // we do this, in case it needs to, for example, adjust values that // // are part of a unique key // ////////////////////////////////////////////////////////////////////// - Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { - preInsertCustomizer.get().setInsertInput(insertInput); - preInsertCustomizer.get().setIsPreview(true); - AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().getWhenToRun(); + AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { - List recordsAfterCustomizer = preInsertCustomizer.get().apply(runBackendStepInput.getRecords()); + List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true); runBackendStepInput.setRecords(recordsAfterCustomizer); /////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index 39dbd7f1..21584e8e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -34,6 +34,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; @@ -252,7 +253,7 @@ public class ColumnStatsStep implements BackendStep QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(); qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null); - QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "count", countField), valueCounts); + QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts); runBackendStepOutput.addValue("valueCounts", valueCounts); @@ -442,13 +443,13 @@ public class ColumnStatsStep implements BackendStep } QFieldMetaData percentField = new QFieldMetaData("percent", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.PERCENT_POINT2).withLabel("Percent"); - QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "percent", percentField), valueCounts); + QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "percent", percentField), valueCounts); } QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null); fields.forEach(qInstanceEnricher::enrichField); - QValueFormatter.setDisplayValuesInRecord(fields, statsRecord); + QValueFormatter.setDisplayValuesInRecord(table, fields.stream().collect(Collectors.toMap(f -> f.getName(), f -> f)), statsRecord); runBackendStepOutput.addValue("statsFields", fields); runBackendStepOutput.addValue("statsRecord", statsRecord); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java index ce227961..516f7d6a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java @@ -112,4 +112,17 @@ public abstract class AbstractExtractStep implements BackendStep this.limit = limit; } + + + /******************************************************************************* + ** Create the record pipe to be used for this process step. + ** + ** Here in case a subclass needs a different type of pipe - for example, a + ** DistinctFilteringRecordPipe. + *******************************************************************************/ + public RecordPipe createRecordPipe(RunBackendStepInput runBackendStepInput, Integer overrideCapacity) + { + return (new RecordPipe(overrideCapacity)); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java index 172ec4cf..51dd30fe 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java @@ -104,12 +104,12 @@ public class BaseStreamedETLStep *******************************************************************************/ protected void moveReviewStepAfterValidateStep(RunBackendStepOutput runBackendStepOutput) { - LOG.info("Skipping to validation step"); + LOG.debug("Skipping to validation step"); ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); - LOG.debug("Step list pre: " + stepList); + LOG.trace("Step list pre: " + stepList); stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); runBackendStepOutput.getProcessState().setStepList(stepList); - LOG.debug("Step list post: " + stepList); + LOG.trace("Step list post: " + stepList); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index 1f3e776d..777d3f72 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -24,8 +24,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.IOException; import java.io.Serializable; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.reporting.DistinctFilteringRecordPipe; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -36,9 +42,13 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -105,6 +115,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep QueryInput queryInput = new QueryInput(); queryInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); queryInput.setFilter(filterClone); + getQueryJoinsForOrderByIfNeeded(queryFilter).forEach(queryJoin -> queryInput.withQueryJoin(queryJoin)); queryInput.setSelectDistinct(true); queryInput.setRecordPipe(getRecordPipe()); queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); @@ -113,6 +124,10 @@ public class ExtractViaQueryStep extends AbstractExtractStep { queryInput.setShouldFetchHeavyFields(true); } + if(runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_INCLUDE_ASSOCIATIONS)) + { + queryInput.setIncludeAssociations(true); + } customizeInputPreQuery(queryInput); @@ -135,6 +150,45 @@ public class ExtractViaQueryStep extends AbstractExtractStep + /******************************************************************************* + ** If the queryFilter has order-by fields from a joinTable, then create QueryJoins + ** for each such table - marked as LEFT, and select=true. + ** + ** This is under the rationale that, the filter would have come from the frontend, + ** which would be doing outer-join semantics for a column being shown (but not filtered by). + ** If the table IS filtered by, it's still OK to do a LEFT, as we'll only get rows + ** that match. + ** + ** Also, they are being select=true'ed so that the DISTINCT clause works (since + ** process queries always try to be DISTINCT). + *******************************************************************************/ + private List getQueryJoinsForOrderByIfNeeded(QQueryFilter queryFilter) + { + if(queryFilter == null) + { + return (Collections.emptyList()); + } + + List rs = new ArrayList<>(); + Set addedTables = new HashSet<>(); + for(QFilterOrderBy filterOrderBy : CollectionUtils.nonNullList(queryFilter.getOrderBys())) + { + if(filterOrderBy.getFieldName().contains(".")) + { + String tableName = filterOrderBy.getFieldName().split("\\.")[0]; + if(!addedTables.contains(tableName)) + { + rs.add(new QueryJoin(tableName).withType(QueryJoin.Type.LEFT).withSelect(true)); + } + addedTables.add(tableName); + } + } + + return (rs); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -144,6 +198,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep CountInput countInput = new CountInput(); countInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); countInput.setFilter(queryFilter); + getQueryJoinsForOrderByIfNeeded(queryFilter).forEach(queryJoin -> countInput.withQueryJoin(queryJoin)); countInput.setIncludeDistinctCount(true); CountOutput countOutput = new CountAction().execute(countInput); Integer count = countOutput.getDistinctCount(); @@ -243,4 +298,33 @@ public class ExtractViaQueryStep extends AbstractExtractStep } } + + + /******************************************************************************* + ** Create the record pipe to be used for this process step. + ** + *******************************************************************************/ + @Override + public RecordPipe createRecordPipe(RunBackendStepInput runBackendStepInput, Integer overrideCapacity) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the filter has order-bys from a join-table, then we have to include that join-table in the SELECT clause, // + // which means we need to do distinct "manually", e.g., via a DistinctFilteringRecordPipe // + // todo - really, wouldn't this only be if it's a many-join? but that's not completely trivial to detect, given join-chains... // + // as it is, we may end up using DistinctPipe in some cases that we need it - which isn't an error, just slightly sub-optimal. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List queryJoinsForOrderByIfNeeded = getQueryJoinsForOrderByIfNeeded(queryFilter); + boolean needDistinctPipe = CollectionUtils.nullSafeHasContents(queryJoinsForOrderByIfNeeded); + + if(needDistinctPipe) + { + String sourceTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE); + QTableMetaData sourceTable = runBackendStepInput.getInstance().getTable(sourceTableName); + return (new DistinctFilteringRecordPipe(new UniqueKey(sourceTable.getPrimaryKeyField()), overrideCapacity)); + } + else + { + return (super.createRecordPipe(runBackendStepInput, overrideCapacity)); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java index 0cee40ac..db39f723 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java @@ -61,7 +61,7 @@ public class LoadViaDeleteStep extends AbstractLoadStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { QTableMetaData table = runBackendStepInput.getTable(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java index f9f87750..7244c07a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java @@ -73,7 +73,7 @@ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { evaluateRecords(runBackendStepInput); insertAndUpdateRecords(runBackendStepInput, runBackendStepOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java index b2fad610..a2db1351 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java @@ -60,7 +60,7 @@ public class LoadViaInsertStep extends AbstractLoadStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { InsertInput insertInput = new InsertInput(); insertInput.setInputSource(getInputSource()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java index 9a9364a9..916f5fb5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java @@ -62,7 +62,7 @@ public class LoadViaUpdateStep extends AbstractLoadStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { UpdateInput updateInput = new UpdateInput(); updateInput.setInputSource(getInputSource()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java index c1333edf..0fcd5abf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java @@ -40,7 +40,7 @@ public class NoopLoadStep extends AbstractLoadStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { /////////// // noop. // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopTransformStep.java index eebacd18..10cf92f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopTransformStep.java @@ -67,7 +67,7 @@ public class NoopTransformStep extends AbstractTransformStep * *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { //////////////////////////////// // return if no input records // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index 3acfe8d4..5c7cc6b0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -58,7 +58,6 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe ** *******************************************************************************/ @Override - @SuppressWarnings("checkstyle:indentation") public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { Optional transaction = Optional.empty(); @@ -136,7 +135,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe asyncRecordPipeLoop.setMinRecordsToConsume(overrideRecordPipeCapacity); } - int recordCount = asyncRecordPipeLoop.run("StreamedETL>Execute>ExtractStep", null, recordPipe, (status) -> + int recordCount = asyncRecordPipeLoop.run("StreamedETLExecute>Extract>" + runBackendStepInput.getProcessName(), null, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); @@ -174,6 +173,14 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe transformStep.postRun(postRunInput, postRunOutput); loadStep.postRun(postRunInput, postRunOutput); + ////////////////////////////////////////////////////////////////////// + // propagate data from inner-step state to process-level step state // + ////////////////////////////////////////////////////////////////////// + if(postRunOutput.getUpdatedFrontendStepList() != null) + { + runBackendStepOutput.setUpdatedFrontendStepList(postRunOutput.getUpdatedFrontendStepList()); + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // explicitly copy values back into the runStepOutput from the post-run output // // this might not be needed, since they (presumably) share a processState object, but just in case that changes... // @@ -268,18 +275,36 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); + transformStep.runOnePage(streamedBackendStepInput, streamedBackendStepOutput); List auditInputListFromTransform = streamedBackendStepOutput.getAuditInputList(); + ////////////////////////////////////////////////////////////////////// + // propagate data from inner-step state to process-level step state // + ////////////////////////////////////////////////////////////////////// + if(streamedBackendStepOutput.getUpdatedFrontendStepList() != null) + { + runBackendStepOutput.getProcessState().setStepList(streamedBackendStepOutput.getProcessState().getStepList()); + runBackendStepOutput.setUpdatedFrontendStepList(streamedBackendStepOutput.getUpdatedFrontendStepList()); + } + //////////////////////////////////////////////// // pass the records through the load function // //////////////////////////////////////////////// streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, streamedBackendStepOutput.getRecords()); streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); - loadStep.run(streamedBackendStepInput, streamedBackendStepOutput); + loadStep.runOnePage(streamedBackendStepInput, streamedBackendStepOutput); List auditInputListFromLoad = streamedBackendStepOutput.getAuditInputList(); + ////////////////////////////////////////////////////////////////////// + // propagate data from inner-step state to process-level step state // + ////////////////////////////////////////////////////////////////////// + if(streamedBackendStepOutput.getUpdatedFrontendStepList() != null) + { + runBackendStepOutput.getProcessState().setStepList(streamedBackendStepOutput.getProcessState().getStepList()); + runBackendStepOutput.setUpdatedFrontendStepList(streamedBackendStepOutput.getUpdatedFrontendStepList()); + } + /////////////////////////////////////////////////////// // copy a small number of records to the output list // /////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index 4921c9ce..04308698 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -72,15 +72,6 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe return; } - ////////////////////////////// - // set up the extract steps // - ////////////////////////////// - AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); - RecordPipe recordPipe = new RecordPipe(); - extractStep.setLimit(limit); - extractStep.setRecordPipe(recordPipe); - extractStep.preRun(runBackendStepInput, runBackendStepOutput); - ///////////////////////////////////////////////////////////////// // if we're running inside an automation, then skip this step. // ///////////////////////////////////////////////////////////////// @@ -90,6 +81,19 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe return; } + ///////////////////////////// + // set up the extract step // + ///////////////////////////// + AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); + extractStep.setLimit(limit); + extractStep.preRun(runBackendStepInput, runBackendStepOutput); + + ////////////////////////////////////////// + // set up a record pipe for the process // + ////////////////////////////////////////// + RecordPipe recordPipe = extractStep.createRecordPipe(runBackendStepInput, null); + extractStep.setRecordPipe(recordPipe); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if skipping frontend steps, skip this action - // // but, if inside an (ideally, only async) API call, at least do the count, so status calls can get x of y status // @@ -125,7 +129,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe // } List previewRecordList = new ArrayList<>(); - new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> + new AsyncRecordPipeLoop().run("StreamedETLPreview>Extract>" + runBackendStepInput.getProcessName(), PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> { runBackendStepInput.setAsyncJobCallback(status); extractStep.run(runBackendStepInput, runBackendStepOutput); @@ -140,6 +144,14 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe BackendStepPostRunOutput postRunOutput = new BackendStepPostRunOutput(runBackendStepOutput); BackendStepPostRunInput postRunInput = new BackendStepPostRunInput(runBackendStepInput); transformStep.postRun(postRunInput, postRunOutput); + + ////////////////////////////////////////////////////////////////////// + // propagate data from inner-step state to process-level step state // + ////////////////////////////////////////////////////////////////////// + if(postRunOutput.getUpdatedFrontendStepList() != null) + { + runBackendStepOutput.setUpdatedFrontendStepList(postRunOutput.getUpdatedFrontendStepList()); + } } @@ -202,7 +214,16 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); + transformStep.runOnePage(streamedBackendStepInput, streamedBackendStepOutput); + + ////////////////////////////////////////////////////////////////////// + // propagate data from inner-step state to process-level step state // + ////////////////////////////////////////////////////////////////////// + if(streamedBackendStepOutput.getUpdatedFrontendStepList() != null) + { + runBackendStepOutput.getProcessState().setStepList(streamedBackendStepOutput.getProcessState().getStepList()); + runBackendStepOutput.setUpdatedFrontendStepList(streamedBackendStepOutput.getUpdatedFrontendStepList()); + } //////////////////////////////////////////////////// // add the transformed records to the output list // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java index ed177c52..c9addb97 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -77,14 +77,16 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back ////////////////////////////////////////////////////////////////////////////////////////////////////////////// moveReviewStepAfterValidateStep(runBackendStepOutput); + AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); + AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); + + runBackendStepInput.getAsyncJobCallback().updateStatus("Validating Records"); + ////////////////////////////////////////////////////////// // basically repeat the preview step, but with no limit // ////////////////////////////////////////////////////////// runBackendStepInput.getAsyncJobCallback().updateStatus("Validating Records"); - AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); - AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); - ////////////////////////////////////////////////////////////////////// // let the transform step override the capacity for the record pipe // ////////////////////////////////////////////////////////////////////// @@ -119,7 +121,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back transformStep.preRun(runBackendStepInput, runBackendStepOutput); List previewRecordList = new ArrayList<>(); - int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Preview>ValidateStep", null, recordPipe, (status) -> + int recordCount = new AsyncRecordPipeLoop().run("StreamedETLValidate>Extract>" + runBackendStepInput.getProcessName(), null, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); @@ -139,6 +141,14 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back BackendStepPostRunOutput postRunOutput = new BackendStepPostRunOutput(runBackendStepOutput); BackendStepPostRunInput postRunInput = new BackendStepPostRunInput(runBackendStepInput); transformStep.postRun(postRunInput, postRunOutput); + + ////////////////////////////////////////////////////////////////////// + // propagate data from inner-step state to process-level step state // + ////////////////////////////////////////////////////////////////////// + if(postRunOutput.getUpdatedFrontendStepList() != null) + { + runBackendStepOutput.setUpdatedFrontendStepList(postRunOutput.getUpdatedFrontendStepList()); + } } @@ -168,7 +178,16 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); + transformStep.runOnePage(streamedBackendStepInput, streamedBackendStepOutput); + + ////////////////////////////////////////////////////////////////////// + // propagate data from inner-step state to process-level step state // + ////////////////////////////////////////////////////////////////////// + if(streamedBackendStepOutput.getUpdatedFrontendStepList() != null) + { + runBackendStepOutput.getProcessState().setStepList(streamedBackendStepOutput.getProcessState().getStepList()); + runBackendStepOutput.setUpdatedFrontendStepList(streamedBackendStepOutput.getUpdatedFrontendStepList()); + } /////////////////////////////////////////////////////// // copy a small number of records to the output list // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 9c279017..8d3e8287 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; @@ -83,6 +84,7 @@ public class StreamedETLWithFrontendProcess public static final String FIELD_RECORD_COUNT = "recordCount"; // Integer public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter) public static final String FIELD_FETCH_HEAVY_FIELDS = "fetchHeavyFields"; // Boolean + public static final String FIELD_INCLUDE_ASSOCIATIONS = "includeAssociations"; // Boolean public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean public static final String FIELD_DO_FULL_VALIDATION = "doFullValidation"; // Boolean @@ -144,6 +146,7 @@ public class StreamedETLWithFrontendProcess .withCode(new QCodeReference(StreamedETLPreviewStep.class)) .withInputData(new QFunctionInputMetaData() .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE))) + .withField(new QFieldMetaData(FIELD_INCLUDE_ASSOCIATIONS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_INCLUDE_ASSOCIATIONS, false))) .withField(new QFieldMetaData(FIELD_FETCH_HEAVY_FIELDS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_FETCH_HEAVY_FIELDS, false))) .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE))) .withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true))) @@ -488,5 +491,28 @@ public class StreamedETLWithFrontendProcess return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java index 4c4830d5..3f4ee4c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorExtractStep.java @@ -26,6 +26,7 @@ import java.time.Instant; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; @@ -57,4 +58,15 @@ public class GarbageCollectorExtractStep extends ExtractViaQueryStep return super.getQueryFilter(runBackendStepInput); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected void customizeInputPreQuery(QueryInput queryInput) + { + queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTransformStep.java index ef233112..b307c975 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTransformStep.java @@ -118,7 +118,7 @@ public class GarbageCollectorTransformStep extends AbstractTransformStep * *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { //////////////////////////////// // return if no input records // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/AbstractMergeDuplicatesTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/AbstractMergeDuplicatesTransformStep.java index 0cd8b71b..98c5d411 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/AbstractMergeDuplicatesTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/AbstractMergeDuplicatesTransformStep.java @@ -203,7 +203,7 @@ public abstract class AbstractMergeDuplicatesTransformStep extends AbstractTrans ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords())) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/MergeDuplicatesLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/MergeDuplicatesLoadStep.java index 572581bd..85470779 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/MergeDuplicatesLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/MergeDuplicatesLoadStep.java @@ -61,9 +61,9 @@ public class MergeDuplicatesLoadStep extends LoadViaInsertOrUpdateStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - super.run(runBackendStepInput, runBackendStepOutput); + super.runOnePage(runBackendStepInput, runBackendStepOutput); ListingHash otherTableIdsToDelete = (ListingHash) runBackendStepInput.getValue("otherTableIdsToDelete"); ListingHash otherTableFiltersToDelete = (ListingHash) runBackendStepInput.getValue("otherTableFiltersToDelete"); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java index 3aad6920..b5128b0c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mock/MockBackendStep.java @@ -56,7 +56,7 @@ public class MockBackendStep implements BackendStep runBackendStepInput.getRecords().forEach(r -> { - LOG.info("We are mocking {}: {}", r.getValueString("firstName"), r.getValue(FIELD_MOCK_VALUE)); + LOG.info("We are mocking " + r.getValueString("firstName") + ": " + r.getValue(FIELD_MOCK_VALUE)); r.setValue(FIELD_MOCK_VALUE, "Ha ha!"); r.setValue("greetingMessage", runBackendStepInput.getValueString(FIELD_GREETING_PREFIX) + " " + r.getValueString("firstName") + " " + runBackendStepInput.getValueString(FIELD_GREETING_SUFFIX)); }); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java index 8d71629e..fdfa381c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; @@ -66,8 +67,9 @@ public class ExecuteReportStep implements BackendStep { ReportInput reportInput = new ReportInput(); reportInput.setReportName(reportName); - reportInput.setReportFormat(ReportFormat.XLSX); // todo - variable - reportInput.setReportOutputStream(reportOutputStream); + reportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.XLSX) // todo - variable + .withReportOutputStream(reportOutputStream)); Map values = runBackendStepInput.getValues(); reportInput.setInputValues(values); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java new file mode 100644 index 00000000..506d2857 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java @@ -0,0 +1,261 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.io.OutputStream; +import java.io.Serializable; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.actions.messaging.SendMessageAction; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Content; +import com.kingsrook.qqq.backend.core.model.actions.messaging.MultiParty; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Party; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailContentRole; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailPartyRole; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReportStatus; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValidationUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Process step to actually execute rendering a saved report. + *******************************************************************************/ +public class RenderSavedReportExecuteStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(RenderSavedReportExecuteStep.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + QRecord renderedReportRecord = null; + + try + { + //////////////////////////////// + // read inputs, set up params // + //////////////////////////////// + String sesProviderName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.SES_PROVIDER_NAME); + String fromEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FROM_EMAIL_ADDRESS); + String replyToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.REPLY_TO_EMAIL_ADDRESS); + String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); + ReportFormat reportFormat = ReportFormat.fromString(runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT)); + String sendToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS); + String emailSubject = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT); + SavedReport savedReport = new SavedReport(runBackendStepInput.getRecords().get(0)); + String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport); + String storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if sending an email (or emails), validate the addresses before doing anything so user gets error and can fix // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List toEmailAddressList = new ArrayList<>(); + if(StringUtils.hasContent(sendToEmailAddress)) + { + toEmailAddressList = ValidationUtils.parseAndValidateEmailAddresses(sendToEmailAddress); + } + + StorageAction storageAction = new StorageAction(); + StorageInput storageInput = new StorageInput(storageTableName).withReference(storageReference); + OutputStream outputStream = storageAction.createOutputStream(storageInput); + + LOG.info("Starting to render a report", logPair("savedReportId", savedReport.getId()), logPair("tableName", savedReport.getTableName()), logPair("storageReference", storageReference)); + runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Report"); + + ////////////////////////////////////////////////////////////////// + // insert a rendered-report record indicating that it's running // + ////////////////////////////////////////////////////////////////// + renderedReportRecord = new InsertAction().execute(new InsertInput(RenderedReport.TABLE_NAME).withRecordEntity(new RenderedReport() + .withSavedReportId(savedReport.getId()) + .withStartTime(Instant.now()) + // todo .withJobUuid(runBackendStepInput.get) + .withRenderedReportStatusId(RenderedReportStatus.RUNNING.getId()) + .withReportFormat(ReportFormatPossibleValueEnum.valueOf(reportFormat.name()).getPossibleValueId()) + )).getRecords().get(0); + + //////////////////////////////////////////////////////////////////////////////////////////// + // convert the report record to report meta-data, which the GenerateReportAction works on // + //////////////////////////////////////////////////////////////////////////////////////////// + QReportMetaData reportMetaData = new SavedReportToReportMetaDataAdapter().adapt(savedReport, reportFormat); + + ///////////////////////////////////// + // setup & run the generate action // + ///////////////////////////////////// + ReportInput reportInput = new ReportInput(); + reportInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); + reportInput.setReportMetaData(reportMetaData); + reportInput.setReportDestination(new ReportDestination() + .withReportFormat(reportFormat) + .withReportOutputStream(outputStream)); + + ////////////////////////////////////////////////////////// + // todo variable-values // + // actually, looks like they're coming in right here... // + ////////////////////////////////////////////////////////// + Map values = runBackendStepInput.getValues(); + reportInput.setInputValues(values); + + ReportOutput reportOutput = new GenerateReportAction().execute(reportInput); + + /////////////////////////////////// + // update record to show success // + /////////////////////////////////// + new UpdateAction().execute(new UpdateInput(RenderedReport.TABLE_NAME).withRecord(new QRecord() + .withValue("id", renderedReportRecord.getValue("id")) + .withValue("resultPath", storageReference) + .withValue("renderedReportStatusId", RenderedReportStatus.COMPLETE.getPossibleValueId()) + .withValue("endTime", Instant.now()) + .withValue("rowCount", reportOutput.getTotalRecordCount()) + )); + + String downloadFileName = downloadFileBaseName + "." + reportFormat.getExtension(); + runBackendStepOutput.addValue("downloadFileName", downloadFileName); + runBackendStepOutput.addValue("storageTableName", storageTableName); + runBackendStepOutput.addValue("storageReference", storageReference); + LOG.info("Completed rendering a report", logPair("savedReportId", savedReport.getId()), logPair("tableName", savedReport.getTableName()), logPair("storageReference", storageReference), logPair("rowCount", reportOutput.getTotalRecordCount())); + + if(!toEmailAddressList.isEmpty() && CollectionUtils.nullSafeHasContents(QContext.getQInstance().getMessagingProviders())) + { + /////////////////////////////////////////// + // error if no from address was provided // + /////////////////////////////////////////// + if(!StringUtils.hasContent(fromEmailAddress)) + { + String message = "Could not send an email because no from email address was provided."; + LOG.error(message); + throw (new QException(message)); + } + + /////////////////////////////////////////////////////////// + // since sending email, make s3 file publicly accessible // + /////////////////////////////////////////////////////////// + storageAction.makePublic(storageInput); + + //////////////////////////////////////////////// + // add multiparty in case multiple recipients // + //////////////////////////////////////////////// + MultiParty recipients = new MultiParty(); + for(String toAddress : toEmailAddressList) + { + recipients.addParty(new Party().withAddress(toAddress).withRole(EmailPartyRole.TO)); + } + + /////////////// + // add froms // + /////////////// + MultiParty froms = new MultiParty(); + froms.addParty(new Party().withAddress(fromEmailAddress).withRole(EmailPartyRole.FROM)); + if(StringUtils.hasContent(replyToEmailAddress)) + { + froms.addParty(new Party().withAddress(replyToEmailAddress).withRole(EmailPartyRole.REPLY_TO)); + } + + String downloadURL = storageAction.getDownloadURL(storageInput); + new SendMessageAction().execute(new SendMessageInput() + .withMessagingProviderName(sesProviderName) + .withTo(recipients) + .withFrom(froms) + .withSubject(StringUtils.hasContent(emailSubject) ? emailSubject : downloadFileBaseName) + .withContent(new Content().withContentRole(EmailContentRole.TEXT).withBody("To download your report, open this URL in your browser: " + downloadURL)) + .withContent(new Content().withContentRole(EmailContentRole.HTML).withBody("Link: " + downloadFileName + "")) + ); + } + } + catch(Exception e) + { + if(renderedReportRecord != null) + { + new UpdateAction().execute(new UpdateInput(RenderedReport.TABLE_NAME).withRecord(new QRecord() + .withValue("id", renderedReportRecord.getValue("id")) + .withValue("renderedReportStatusId", RenderedReportStatus.FAILED.getPossibleValueId()) + .withValue("endTime", Instant.now()) + .withValue("errorMessage", ExceptionUtils.concatenateMessagesFromChain(e)) + )); + } + + LOG.warn("Error rendering saved report", e); + throw (e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report) + { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmm").withZone(ZoneId.systemDefault()); + String datePart = formatter.format(Instant.now()); + + String downloadFileBaseName = runBackendStepInput.getValueString("downloadFileBaseName"); + if(!StringUtils.hasContent(downloadFileBaseName)) + { + downloadFileBaseName = report.getLabel(); + } + + ////////////////////////////////////////////////// + // these chars have caused issues, so, disallow // + ////////////////////////////////////////////////// + downloadFileBaseName = downloadFileBaseName.replaceAll("/", "-").replaceAll(",", "_"); + + return (downloadFileBaseName + " - " + datePart); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java new file mode 100644 index 00000000..10bc4212 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java @@ -0,0 +1,116 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; + + +/******************************************************************************* + ** define process for rendering saved reports! + *******************************************************************************/ +public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterface +{ + public static final String NAME = "renderSavedReport"; + + public static final String SES_PROVIDER_NAME = "sesProviderName"; + public static final String FROM_EMAIL_ADDRESS = "fromEmailAddress"; + public static final String REPLY_TO_EMAIL_ADDRESS = "replyToEmailAddress"; + public static final String FIELD_NAME_STORAGE_TABLE_NAME = "storageTableName"; + public static final String FIELD_NAME_REPORT_FORMAT = "reportFormat"; + public static final String FIELD_NAME_EMAIL_ADDRESS = "reportDestinationEmailAddress"; + public static final String FIELD_NAME_EMAIL_SUBJECT = "emailSubject"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + QProcessMetaData process = new QProcessMetaData() + .withName(NAME) + .withLabel("Render Report") + .withTableName(SavedReport.TABLE_NAME) + .withIcon(new QIcon().withName("print")) + + .addStep(new QBackendStepMetaData() + .withName("pre") + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData(SES_PROVIDER_NAME, QFieldType.STRING)) + .withField(new QFieldMetaData(FROM_EMAIL_ADDRESS, QFieldType.STRING)) + .withField(new QFieldMetaData(REPLY_TO_EMAIL_ADDRESS, QFieldType.STRING)) + .withField(new QFieldMetaData(FIELD_NAME_STORAGE_TABLE_NAME, QFieldType.STRING)) + .withRecordListMetaData(new QRecordListMetaData().withTableName(SavedReport.TABLE_NAME))) + .withCode(new QCodeReference(RenderSavedReportPreStep.class))) + + .addStep(new QFrontendStepMetaData() + .withName("input") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) + .withFormField(new QFieldMetaData(FIELD_NAME_REPORT_FORMAT, QFieldType.STRING) + .withPossibleValueSourceName(ReportFormatPossibleValueEnum.NAME) + .withIsRequired(true)) + .withFormField(new QFieldMetaData(FIELD_NAME_EMAIL_ADDRESS, QFieldType.STRING).withLabel("Email To")) + .withFormField(new QFieldMetaData(FIELD_NAME_EMAIL_SUBJECT, QFieldType.STRING).withLabel("Email Subject")) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.WIDGET) + .withValue("widgetName", SavedReportsMetaDataProvider.RENDER_REPORT_PROCESS_VALUES_WIDGET))) + + .addStep(new QBackendStepMetaData() + .withName("execute") + .withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData() + .withTableName(SavedReport.TABLE_NAME))) + .withCode(new QCodeReference(RenderSavedReportExecuteStep.class))) + + .addStep(new QFrontendStepMetaData() + .withName("output") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.DOWNLOAD_FORM)) + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine() + .withCondition(new QFilterCriteria(FIELD_NAME_EMAIL_ADDRESS, QCriteriaOperator.IS_NOT_BLANK)) + .withVelocityTemplate(String.format("Report has been emailed to: ${%s}", FIELD_NAME_EMAIL_ADDRESS)))) + ); + + return (process); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportPreStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportPreStep.java new file mode 100644 index 00000000..3f999088 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportPreStep.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** initial backend-step before rendering a saved report. does some basic + ** validations, and then (in future) will set up input fields (how??) for the + ** input screen. + *******************************************************************************/ +public class RenderSavedReportPreStep implements BackendStep +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); + if(!StringUtils.hasContent(storageTableName)) + { + throw (new QUserFacingException("Process configuration error: Missing value for storageTableName.")); + } + + if(QContext.getQInstance().getTable(storageTableName) == null) + { + throw (new QUserFacingException("Process configuration error: Unrecognized value for storageTableName - no table named [" + storageTableName + "] was found in the instance.")); + } + + List records = runBackendStepInput.getRecords(); + if(!CollectionUtils.nullSafeHasContents(records)) + { + throw (new QUserFacingException("No report was selected or found.")); + } + + if(records.size() > 1) + { + throw (new QUserFacingException("You may only run 1 report at a time.")); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // put the savedReportId in values - this'll get passed into the widget, so it knows // + // what report we're working with, and thus what inputs to prompt for // + // also put a value in just to help it know we're running the process // + /////////////////////////////////////////////////////////////////////////////////////// + SavedReport savedReport = new SavedReport(records.get(0)); + runBackendStepOutput.addValue("savedReportId", savedReport.getId()); + runBackendStepOutput.addValue("processName", runBackendStepInput.getProcessName()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RunScheduledReportExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RunScheduledReportExecuteStep.java new file mode 100644 index 00000000..8ad4b673 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RunScheduledReportExecuteStep.java @@ -0,0 +1,153 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.savedreports.ScheduledReport; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.json.JSONObject; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RunScheduledReportExecuteStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(RunScheduledReportExecuteStep.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + Integer scheduledReportId = null; + try + { + List records = runBackendStepInput.getRecords(); + if(!CollectionUtils.nullSafeHasContents(records)) + { + throw (new QUserFacingException("No scheduled report was selected or found.")); + } + + ScheduledReport scheduledReport = new ScheduledReport(records.get(0)); + scheduledReportId = scheduledReport.getId(); + + //////////////////////////////////////////////////////////////////////////////////// + // get the schedule's user - as that will drive the security key we need to apply // + //////////////////////////////////////////////////////////////////////////////////// + updateSessionForUser(scheduledReport.getUserId()); + + ///////////////////////////////////////////// + // run the process that renders the report // + ///////////////////////////////////////////// + RunProcessAction runProcessAction = new RunProcessAction(); + RunProcessInput renderProcessInput = new RunProcessInput(); + renderProcessInput.setProcessName(RenderSavedReportMetaDataProducer.NAME); + renderProcessInput.setCallback(QProcessCallbackFactory.forPrimaryKey("id", scheduledReport.getSavedReportId())); + renderProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + renderProcessInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); + + renderProcessInput.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, scheduledReport.getFormat()); + renderProcessInput.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS, scheduledReport.getToAddresses()); + renderProcessInput.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT, scheduledReport.getSubject()); + + if(StringUtils.hasContent(scheduledReport.getInputValues())) + { + ///////////////////////////////////////////////////////////////////////////////////// + // if there are input values, pass them along on report input... // + // this could maybe be better (e.g., some object?), but, this is working initially // + ///////////////////////////////////////////////////////////////////////////////////// + JSONObject jsonObject = JsonUtils.toJSONObject(scheduledReport.getInputValues()); + for(String name : jsonObject.keySet()) + { + renderProcessInput.addValue(name, jsonObject.optString(name)); + } + } + + RunProcessOutput renderProcessOutput = runProcessAction.execute(renderProcessInput); + } + catch(QUserFacingException ufe) + { + LOG.info("Error running scheduled report", ufe, logPair("id", scheduledReportId)); + throw (ufe); + } + catch(Exception e) + { + LOG.warn("Error running scheduled report", e, logPair("id", scheduledReportId)); + throw (new QException("Error running scheduled report", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void updateSessionForUser(String userId) throws QException + { + try + { + QInstance qInstance = QContext.getQInstance(); + QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher(); + QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(qInstance.getAuthentication()); + + /////////////////////////////////////// + // create automated-session for user // + /////////////////////////////////////// + QSession session = authenticationModule.createAutomatedSessionForUser(qInstance, userId); + + ///////////////////////////////////////////// + // set that session in the current context // + ///////////////////////////////////////////// + QContext.setQSession(session); + } + catch(Exception e) + { + LOG.warn("Error setting up user session for running scheduled report", e, logPair("userId", userId)); + throw (new QException("Error setting up user session for running scheduled report", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RunScheduledReportMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RunScheduledReportMetaDataProducer.java new file mode 100644 index 00000000..15717f22 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RunScheduledReportMetaDataProducer.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.ScheduledReport; + + +/******************************************************************************* + ** define process for rendering scheduled reports - that is - a thin layer on + ** top of rendering a saved report. + *******************************************************************************/ +public class RunScheduledReportMetaDataProducer implements MetaDataProducerInterface +{ + public static final String NAME = "runScheduledReport"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + QProcessMetaData process = new QProcessMetaData() + .withName(NAME) + .withLabel("Run Scheduled Report") + .withTableName(ScheduledReport.TABLE_NAME) + .withIcon(new QIcon().withName("print")) + + .addStep(new QBackendStepMetaData() + .withName("execute") + .withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData() + .withTableName(ScheduledReport.TABLE_NAME))) + .withCode(new QCodeReference(RunScheduledReportExecuteStep.class))) + + .addStep(new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Success")))); // todo!!! + + return (process); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java new file mode 100644 index 00000000..55ab7270 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java @@ -0,0 +1,408 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumn; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** GenerateReportAction takes in ReportMetaData. + ** + ** This class knows how to adapt from a SavedReport to a ReportMetaData, so that + ** we can render a saved report. + *******************************************************************************/ +public class SavedReportToReportMetaDataAdapter +{ + private static final QLogger LOG = QLogger.getLogger(SavedReportToReportMetaDataAdapter.class); + + private static Consumer jsonMapperCustomizer = om -> om.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QReportMetaData adapt(SavedReport savedReport, ReportFormat reportFormat) throws QException + { + try + { + QInstance qInstance = QContext.getQInstance(); + + QReportMetaData reportMetaData = new QReportMetaData(); + reportMetaData.setName("savedReport:" + savedReport.getId()); + reportMetaData.setLabel(savedReport.getLabel()); + + ///////////////////////////////////////////////////// + // set up the data-source - e.g., table and filter // + ///////////////////////////////////////////////////// + QReportDataSource dataSource = new QReportDataSource(); + reportMetaData.setDataSources(List.of(dataSource)); + dataSource.setName("main"); + + QTableMetaData table = qInstance.getTable(savedReport.getTableName()); + dataSource.setSourceTable(savedReport.getTableName()); + dataSource.setQueryFilter(getQQueryFilter(savedReport.getQueryFilterJson())); + + ////////////////////////// + // set up the main view // + ////////////////////////// + QReportView view = new QReportView(); + reportMetaData.setViews(ListBuilder.of(view)); + view.setName("main"); + view.setType(ReportType.TABLE); + view.setDataSourceName(dataSource.getName()); + view.setLabel(savedReport.getLabel()); + view.setIncludeHeaderRow(true); + + /////////////////////////////////////////////////////////////////////////////////////////////// + // columns in the saved-report should look like a serialized version of ReportColumns object // + // map them to a list of QReportField objects // + // also keep track of what joinTables we find that we need to select // + /////////////////////////////////////////////////////////////////////////////////////////////// + ReportColumns columnsObject = getReportColumns(savedReport.getColumnsJson()); + + List reportColumns = new ArrayList<>(); + view.setColumns(reportColumns); + + Set neededJoinTables = new HashSet<>(); + + for(ReportColumn column : columnsObject.extractVisibleColumns()) + { + //////////////////////////////////////////////////// + // figure out the field being named by the column // + //////////////////////////////////////////////////// + String fieldName = column.getName(); + FieldAndJoinTable fieldAndJoinTable = getField(savedReport, fieldName, qInstance, neededJoinTables, table); + if(fieldAndJoinTable == null) + { + continue; + } + + ////////////////////////////////////////////////// + // make a QReportField based on the table field // + ////////////////////////////////////////////////// + reportColumns.add(makeQReportField(fieldName, fieldAndJoinTable)); + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // set up joins, if we need any // + // note - test coverage here is provided by RDBMS module's GenerateReportActionRDBMSTest // + /////////////////////////////////////////////////////////////////////////////////////////// + if(!neededJoinTables.isEmpty()) + { + List queryJoins = new ArrayList<>(); + dataSource.setQueryJoins(queryJoins); + + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + if(neededJoinTables.contains(exposedJoin.getJoinTable())) + { + QueryJoin queryJoin = new QueryJoin(exposedJoin.getJoinTable()) + .withSelect(true) + .withType(QueryJoin.Type.LEFT) + .withBaseTableOrAlias(null) + .withAlias(null); + + if(exposedJoin.getJoinPath().size() == 1) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Note, this is similar logic (and comment) in QFMD ... // + // todo - what about a join with a longer path? it would be nice to pass such joinNames through there too, // + // but what, that would actually be multiple queryJoins? needs a fair amount of thought. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + queryJoin.setJoinMetaData(qInstance.getJoin(exposedJoin.getJoinPath().get(0))); + } + + queryJoins.add(queryJoin); + } + } + } + + ///////////////////////////////////////// + // if it's a pivot report, handle that // + ///////////////////////////////////////// + if(StringUtils.hasContent(savedReport.getPivotTableJson())) + { + PivotTableDefinition pivotTableDefinition = getPivotTableDefinition(savedReport.getPivotTableJson()); + + QReportView pivotView = new QReportView(); + reportMetaData.getViews().add(pivotView); + pivotView.setName("pivot"); + pivotView.setLabel("Pivot Table"); + + if(reportFormat != null && reportFormat.getSupportsNativePivotTables()) + { + pivotView.setType(ReportType.PIVOT); + pivotView.setPivotTableSourceViewName(view.getName()); + pivotView.setPivotTableDefinition(pivotTableDefinition); + } + else + { + if(!CollectionUtils.nullSafeHasContents(pivotTableDefinition.getRows())) + { + throw (new QUserFacingException("To generate a pivot report in " + reportFormat + " format, it must have 1 or more Pivot Rows")); + } + + if(CollectionUtils.nullSafeHasContents(pivotTableDefinition.getColumns())) + { + throw (new QUserFacingException("To generate a pivot report in " + reportFormat + " format, it may not have any Pivot Columns")); + } + + /////////////////////// + // handle pivot rows // + /////////////////////// + List summaryFields = new ArrayList<>(); + List summaryOrderByFields = new ArrayList<>(); + for(PivotTableGroupBy row : pivotTableDefinition.getRows()) + { + String fieldName = row.getFieldName(); + FieldAndJoinTable fieldAndJoinTable = getField(savedReport, fieldName, qInstance, neededJoinTables, table); + if(fieldAndJoinTable == null) + { + LOG.warn("The field for a Pivot Row wasn't found, when converting to a summary...", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName)); + continue; + } + summaryFields.add(fieldName); + summaryOrderByFields.add(new QFilterOrderBy(fieldName)); + } + + ///////////////////////// + // handle pivot values // + ///////////////////////// + List summaryViewColumns = new ArrayList<>(); + for(PivotTableValue value : pivotTableDefinition.getValues()) + { + String fieldName = value.getFieldName(); + FieldAndJoinTable fieldAndJoinTable = getField(savedReport, fieldName, qInstance, neededJoinTables, table); + if(fieldAndJoinTable == null) + { + LOG.warn("The field for a Pivot Value wasn't found, when converting to a summary...", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName)); + continue; + } + + QReportField reportField = makeQReportField(fieldName, fieldAndJoinTable); + reportField.setName(fieldName + "_" + value.getFunction().name()); + reportField.setLabel(StringUtils.ucFirst(value.getFunction().name().toLowerCase()) + " Of " + reportField.getLabel()); + reportField.setFormula("${pivot." + value.getFunction().name().toLowerCase() + "." + fieldName + "}"); + summaryViewColumns.add(reportField); + summaryOrderByFields.add(new QFilterOrderBy(reportField.getName())); + } + + pivotView.setType(ReportType.SUMMARY); + pivotView.setDataSourceName(dataSource.getName()); + pivotView.setIncludeHeaderRow(true); + pivotView.setIncludeTotalRow(true); + pivotView.setColumns(summaryViewColumns); + pivotView.setSummaryFields(summaryFields); + pivotView.withOrderByFields(summaryOrderByFields); + } + + //////////////////////////////////////////////////////////////////////////////////// + // in case the reportFormat doesn't support multiple views, and we have a pivot - // + // then remove the data view // + //////////////////////////////////////////////////////////////////////////////////// + if(reportFormat != null && !reportFormat.getSupportsMultipleViews()) + { + reportMetaData.getViews().remove(0); + } + } + + ///////////////////////////////////////////////////// + // add input fields, if they're in the savedReport // + ///////////////////////////////////////////////////// + if(StringUtils.hasContent(savedReport.getInputFieldsJson())) + { + //////////////////////////////////// + // todo turn on when implementing // + //////////////////////////////////// + // reportMetaData.setInputFields(JsonUtils.toObject(savedReport.getInputFieldsJson(), new TypeReference<>() {}), objectMapperConsumer); + throw (new IllegalStateException("Input Fields are not yet implemented")); + } + + return (reportMetaData); + } + catch(Exception e) + { + LOG.warn("Error adapting savedReport to reportMetaData", e, logPair("savedReportId", savedReport.getId())); + throw (new QException("Error adapting savedReport to reportMetaData", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static PivotTableDefinition getPivotTableDefinition(String pivotTableJson) throws IOException + { + return JsonUtils.toObject(pivotTableJson, PivotTableDefinition.class, jsonMapperCustomizer); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ReportColumns getReportColumns(String columnsJson) throws IOException + { + return JsonUtils.toObject(columnsJson, ReportColumns.class, jsonMapperCustomizer); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QQueryFilter getQQueryFilter(String queryFilterJson) throws IOException + { + return JsonUtils.toObject(queryFilterJson, QQueryFilter.class, jsonMapperCustomizer); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportField makeQReportField(String fieldName, FieldAndJoinTable fieldAndJoinTable) + { + QReportField reportField = new QReportField(); + + reportField.setName(fieldName); + + if(fieldAndJoinTable.joinTable() == null) + { + //////////////////////////////////////////////////////////// + // for fields from this table, just use the field's label // + //////////////////////////////////////////////////////////// + reportField.setLabel(fieldAndJoinTable.field().getLabel()); + } + else + { + /////////////////////////////////////////////////////////////// + // for fields from join tables, use table label: field label // + /////////////////////////////////////////////////////////////// + reportField.setLabel(fieldAndJoinTable.joinTable().getLabel() + ": " + fieldAndJoinTable.field().getLabel()); + } + + if(StringUtils.hasContent(fieldAndJoinTable.field().getPossibleValueSourceName())) + { + reportField.setShowPossibleValueLabel(true); + } + + reportField.setType(fieldAndJoinTable.field().getType()); + + return reportField; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static FieldAndJoinTable getField(SavedReport savedReport, String fieldName, QInstance qInstance, Set neededJoinTables, QTableMetaData table) + { + QFieldMetaData field; + if(fieldName.contains(".")) + { + String joinTableName = fieldName.replaceAll("\\..*", ""); + String joinFieldName = fieldName.replaceAll(".*\\.", ""); + + QTableMetaData joinTable = qInstance.getTable(joinTableName); + if(joinTable == null) + { + LOG.warn("Saved Report has an unrecognized join table name", logPair("savedReportId", savedReport.getId()), logPair("joinTable", joinTable), logPair("fieldName", fieldName)); + return null; + } + + neededJoinTables.add(joinTableName); + + field = joinTable.getFields().get(joinFieldName); + if(field == null) + { + LOG.warn("Saved Report has an unrecognized join field name", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName)); + return null; + } + + return new FieldAndJoinTable(field, joinTable); + } + else + { + field = table.getFields().get(fieldName); + if(field == null) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // frontend may often pass __checked__ (or maybe other __ prefixes in the future - so - don't warn that. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!fieldName.startsWith("__")) + { + LOG.warn("Saved Report has an unexpected unrecognized field name", logPair("savedReportId", savedReport.getId()), logPair("table", table.getName()), logPair("fieldName", fieldName)); + } + return null; + } + + return new FieldAndJoinTable(field, null); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) {} +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java index 180f7877..48eebc17 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java @@ -113,7 +113,7 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { runBackendStepInput.getAsyncJobCallback().updateStatus("Running script"); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptTransformStep.java index 1eb21e7b..1718b79e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptTransformStep.java @@ -88,9 +88,9 @@ public class RunRecordScriptTransformStep extends NoopTransformStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - super.run(runBackendStepInput, runBackendStepOutput); + super.runOnePage(runBackendStepInput, runBackendStepOutput); runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, doGetProcessSummary(runBackendStepOutput, false)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/DeleteSharedRecordProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/DeleteSharedRecordProcess.java new file mode 100644 index 00000000..ea73e9ff --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/DeleteSharedRecordProcess.java @@ -0,0 +1,130 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** DeleteSharedRecord: {tableName; recordId; shareId;} + *******************************************************************************/ +public class DeleteSharedRecordProcess implements BackendStep, MetaDataProducerInterface +{ + public static final String NAME = "deleteSharedRecord"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(NAME) + .withIcon(new QIcon().withName("share")) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)) // todo confirm or protect + .withStepList(List.of( + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) // todo - actually only a subset of this... + .withField(new QFieldMetaData("recordId", QFieldType.STRING)) + .withField(new QFieldMetaData("shareId", QFieldType.INTEGER)) + ) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + String tableName = runBackendStepInput.getValueString("tableName"); + String recordIdString = runBackendStepInput.getValueString("recordId"); + Integer shareId = runBackendStepInput.getValueInteger("shareId"); + + Objects.requireNonNull(tableName, "Missing required input: tableName"); + Objects.requireNonNull(recordIdString, "Missing required input: recordId"); + Objects.requireNonNull(shareId, "Missing required input: shareId"); + + try + { + SharedRecordProcessUtils.AssetTableAndRecord assetTableAndRecord = SharedRecordProcessUtils.getAssetTableAndRecord(tableName, recordIdString); + + ShareableTableMetaData shareableTableMetaData = assetTableAndRecord.shareableTableMetaData(); + QRecord assetRecord = assetTableAndRecord.record(); + + SharedRecordProcessUtils.assertRecordOwnership(shareableTableMetaData, assetRecord, "delete shares of"); + + /////////////////// + // do the delete // + /////////////////// + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(shareableTableMetaData.getSharedRecordTableName()).withPrimaryKeys(List.of(shareId))); + + ////////////////////// + // check for errors // + ////////////////////// + if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors())) + { + throw (new QException("Error deleting shared record: " + deleteOutput.getRecordsWithErrors().get(0).getErrors().get(0).getMessage())); + } + } + catch(QException qe) + { + throw (qe); + } + catch(Exception e) + { + throw (new QException("Error deleting shared record", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/EditSharedRecordProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/EditSharedRecordProcess.java new file mode 100644 index 00000000..406bea55 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/EditSharedRecordProcess.java @@ -0,0 +1,140 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** EditSharedRecord: {tableName; recordId; shareId; scopeId;} + *******************************************************************************/ +public class EditSharedRecordProcess implements BackendStep, MetaDataProducerInterface +{ + public static final String NAME = "editSharedRecord"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(NAME) + .withIcon(new QIcon().withName("share")) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)) // todo confirm or protect + .withStepList(List.of( + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) // todo - actually only a subset of this... + .withField(new QFieldMetaData("recordId", QFieldType.STRING)) + .withField(new QFieldMetaData("scopeId", QFieldType.STRING).withPossibleValueSourceName(ShareScopePossibleValueMetaDataProducer.NAME)) + .withField(new QFieldMetaData("shareId", QFieldType.INTEGER)) + ) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + String tableName = runBackendStepInput.getValueString("tableName"); + String recordIdString = runBackendStepInput.getValueString("recordId"); + String scopeId = runBackendStepInput.getValueString("scopeId"); + Integer shareId = runBackendStepInput.getValueInteger("shareId"); + + Objects.requireNonNull(tableName, "Missing required input: tableName"); + Objects.requireNonNull(recordIdString, "Missing required input: recordId"); + Objects.requireNonNull(scopeId, "Missing required input: scopeId"); + Objects.requireNonNull(shareId, "Missing required input: shareId"); + + try + { + SharedRecordProcessUtils.AssetTableAndRecord assetTableAndRecord = SharedRecordProcessUtils.getAssetTableAndRecord(tableName, recordIdString); + + ShareableTableMetaData shareableTableMetaData = assetTableAndRecord.shareableTableMetaData(); + QRecord assetRecord = assetTableAndRecord.record(); + QTableMetaData shareTable = QContext.getQInstance().getTable(shareableTableMetaData.getSharedRecordTableName()); + + SharedRecordProcessUtils.assertRecordOwnership(shareableTableMetaData, assetRecord, "edit shares of"); + ShareScope shareScope = SharedRecordProcessUtils.validateScopeId(scopeId); + + /////////////////// + // do the insert // + /////////////////// + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(shareableTableMetaData.getSharedRecordTableName()).withRecord(new QRecord() + .withValue(shareTable.getPrimaryKeyField(), shareId) + .withValue(shareableTableMetaData.getScopeFieldName(), shareScope.getPossibleValueId()))); + + ////////////////////// + // check for errors // + ////////////////////// + if(CollectionUtils.nullSafeHasContents(updateOutput.getRecords().get(0).getErrors())) + { + throw (new QException("Error editing shared record: " + updateOutput.getRecords().get(0).getErrors().get(0).getMessage())); + } + } + catch(QException qe) + { + throw (qe); + } + catch(Exception e) + { + throw (new QException("Error editing shared record", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/GetSharedRecordsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/GetSharedRecordsProcess.java new file mode 100644 index 00000000..5bf42978 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/GetSharedRecordsProcess.java @@ -0,0 +1,249 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** GetSharedRecords: {tableName; recordId;} => [{id; audienceType; audienceId; audienceLabel; scopeId}] + *******************************************************************************/ +public class GetSharedRecordsProcess implements BackendStep, MetaDataProducerInterface +{ + public static final String NAME = "getSharedRecords"; + + private static final QLogger LOG = QLogger.getLogger(GetSharedRecordsProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(NAME) + .withIcon(new QIcon().withName("share")) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)) // todo confirm or protect + .withStepList(List.of( + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) // todo - actually only a subset of this... + .withField(new QFieldMetaData("recordId", QFieldType.STRING)) + ) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + String tableName = runBackendStepInput.getValueString("tableName"); + String recordIdString = runBackendStepInput.getValueString("recordId"); + + Objects.requireNonNull(tableName, "Missing required input: tableName"); + Objects.requireNonNull(recordIdString, "Missing required input: recordId"); + + try + { + SharedRecordProcessUtils.AssetTableAndRecord assetTableAndRecord = SharedRecordProcessUtils.getAssetTableAndRecord(tableName, recordIdString); + + ShareableTableMetaData shareableTableMetaData = assetTableAndRecord.shareableTableMetaData(); + QTableMetaData shareTable = QContext.getQInstance().getTable(shareableTableMetaData.getSharedRecordTableName()); + Serializable recordId = assetTableAndRecord.recordId(); + + ///////////////////////////////////// + // query for shares on this record // + ///////////////////////////////////// + QueryInput queryInput = new QueryInput(shareTable.getName()); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria(shareableTableMetaData.getAssetIdFieldName(), QCriteriaOperator.EQUALS, recordId)) + .withOrderBy(new QFilterOrderBy(shareTable.getPrimaryKeyField())) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // iterate results, building QRecords to output - note - we'll need to collect ids, then look them up in audience-source tables // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ArrayList resultList = new ArrayList<>(); + ListingHash audienceIds = new ListingHash<>(); + for(QRecord record : queryOutput.getRecords()) + { + QRecord outputRecord = new QRecord(); + outputRecord.setValue("shareId", record.getValue(shareTable.getPrimaryKeyField())); + outputRecord.setValue("scopeId", record.getValue(shareableTableMetaData.getScopeFieldName())); + + boolean foundAudienceType = false; + for(ShareableAudienceType audienceType : shareableTableMetaData.getAudienceTypes().values()) + { + Serializable audienceId = record.getValue(audienceType.getFieldName()); + if(audienceId != null) + { + outputRecord.setValue("audienceType", audienceType.getName()); + outputRecord.setValue("audienceId", audienceId); + audienceIds.add(audienceType.getName(), audienceId); + foundAudienceType = true; + break; + } + } + + if(!foundAudienceType) + { + LOG.warn("Failed to find what audience type to use for a shared record", + logPair("sharedTableName", shareTable.getName()), + logPair("id", record.getValue(shareTable.getPrimaryKeyField())), + logPair("recordId", record.getValue(shareableTableMetaData.getAssetIdFieldName()))); + continue; + } + + resultList.add(outputRecord); + } + + ///////////////////////////////// + // look up the audience labels // + ///////////////////////////////// + Map> audienceLabels = new HashMap<>(); + Set audienceTypesWithLabels = new HashSet<>(); + for(Map.Entry> entry : audienceIds.entrySet()) + { + String audienceType = entry.getKey(); + List ids = entry.getValue(); + if(CollectionUtils.nullSafeHasContents(ids)) + { + ShareableAudienceType shareableAudienceType = shareableTableMetaData.getAudienceTypes().get(audienceType); + if(StringUtils.hasContent(shareableAudienceType.getSourceTableName())) + { + audienceTypesWithLabels.add(audienceType); + + String keyField = shareableAudienceType.getSourceTableKeyFieldName(); + if(!StringUtils.hasContent(keyField)) + { + keyField = QContext.getQInstance().getTable(shareableAudienceType.getSourceTableName()).getPrimaryKeyField(); + } + + QueryInput audienceQueryInput = new QueryInput(shareableAudienceType.getSourceTableName()); + audienceQueryInput.setFilter(new QQueryFilter(new QFilterCriteria(keyField, QCriteriaOperator.IN, ids))); + audienceQueryInput.setShouldGenerateDisplayValues(true); // to get record labels + QueryOutput audienceQueryOutput = new QueryAction().execute(audienceQueryInput); + for(QRecord audienceRecord : audienceQueryOutput.getRecords()) + { + audienceLabels.computeIfAbsent(audienceType, k -> new HashMap<>()); + audienceLabels.get(audienceType).put(audienceRecord.getValue(keyField), audienceRecord.getRecordLabel()); + } + } + } + } + + //////////////////////////////////////////// + // put those labels on the output records // + //////////////////////////////////////////// + for(QRecord outputRecord : resultList) + { + String audienceType = outputRecord.getValueString("audienceType"); + Map typeLabels = audienceLabels.getOrDefault(audienceType, Collections.emptyMap()); + Serializable audienceId = outputRecord.getValue("audienceId"); + String label = typeLabels.get(audienceId); + if(StringUtils.hasContent(label)) + { + outputRecord.setValue("audienceLabel", label); + } + else + { + if(audienceTypesWithLabels.contains(audienceType)) + { + outputRecord.setValue("audienceLabel", "Unknown " + audienceType + " (id=" + audienceId + ")"); + } + else + { + outputRecord.setValue("audienceLabel", audienceType + " " + audienceId); + } + } + } + + //////////////////////////// + // sort results by labels // + //////////////////////////// + resultList.sort(Comparator.comparing(r -> r.getValueString("audienceLabel"))); + + runBackendStepOutput.addValue("resultList", resultList); + } + catch(QException qe) + { + throw (qe); + } + catch(Exception e) + { + throw (new QException("Error getting shared records.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/InsertSharedRecordProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/InsertSharedRecordProcess.java new file mode 100644 index 00000000..c3ba8688 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/InsertSharedRecordProcess.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.DuplicateKeyBadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** InsertSharedRecord: {tableName; recordId; audienceType; audienceId; scopeId;} + *******************************************************************************/ +public class InsertSharedRecordProcess implements BackendStep, MetaDataProducerInterface +{ + public static final String NAME = "insertSharedRecord"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(NAME) + .withIcon(new QIcon().withName("share")) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)) // todo confirm or protect + .withStepList(List.of( + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) // todo - actually only a subset of this... + .withField(new QFieldMetaData("recordId", QFieldType.STRING)) + .withField(new QFieldMetaData("audienceType", QFieldType.STRING)) // todo take a PVS name as param? + .withField(new QFieldMetaData("audienceId", QFieldType.STRING)) + .withField(new QFieldMetaData("scopeId", QFieldType.STRING).withPossibleValueSourceName(ShareScopePossibleValueMetaDataProducer.NAME)) + ) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + String tableName = runBackendStepInput.getValueString("tableName"); + String recordIdString = runBackendStepInput.getValueString("recordId"); + String audienceType = runBackendStepInput.getValueString("audienceType"); + String audienceIdString = runBackendStepInput.getValueString("audienceId"); + String scopeId = runBackendStepInput.getValueString("scopeId"); + + Objects.requireNonNull(tableName, "Missing required input: tableName"); + Objects.requireNonNull(recordIdString, "Missing required input: recordId"); + Objects.requireNonNull(audienceType, "Missing required input: audienceType"); + Objects.requireNonNull(audienceIdString, "Missing required input: audienceId"); + Objects.requireNonNull(scopeId, "Missing required input: scopeId"); + + String assetTableLabel = tableName; + try + { + SharedRecordProcessUtils.AssetTableAndRecord assetTableAndRecord = SharedRecordProcessUtils.getAssetTableAndRecord(tableName, recordIdString); + + ShareableTableMetaData shareableTableMetaData = assetTableAndRecord.shareableTableMetaData(); + QRecord assetRecord = assetTableAndRecord.record(); + Serializable recordId = assetTableAndRecord.recordId(); + assetTableLabel = assetTableAndRecord.table().getLabel(); + + SharedRecordProcessUtils.assertRecordOwnership(shareableTableMetaData, assetRecord, "share"); + + //////////////////////////////// + // validate the audience type // + //////////////////////////////// + ShareableAudienceType shareableAudienceType = shareableTableMetaData.getAudienceTypes().get(audienceType); + if(shareableAudienceType == null) + { + throw (new QException("[" + audienceType + "] is not a recognized audience type for sharing records from the " + tableName + " table. Allowed values are: " + shareableTableMetaData.getAudienceTypes().keySet())); + } + + /////////////////////////////////////////////////////////////////////////////////////////////// + // if we know the audience source-table, then fetch & validate security-wise the audience id // + /////////////////////////////////////////////////////////////////////////////////////////////// + Serializable audienceId = audienceIdString; + String audienceTableLabel = "audience"; + if(StringUtils.hasContent(shareableAudienceType.getSourceTableName())) + { + QTableMetaData audienceTable = QContext.getQInstance().getTable(shareableAudienceType.getSourceTableName()); + audienceTableLabel = audienceTable.getLabel(); + + GetInput getInput = new GetInput(audienceTable.getName()); + if(StringUtils.hasContent(shareableAudienceType.getSourceTableKeyFieldName())) + { + audienceId = ValueUtils.getValueAsFieldType(audienceTable.getField(shareableAudienceType.getSourceTableKeyFieldName()).getType(), audienceIdString); + getInput.withUniqueKey(Map.of(shareableAudienceType.getSourceTableKeyFieldName(), audienceId)); + } + else + { + audienceId = ValueUtils.getValueAsFieldType(audienceTable.getField(audienceTable.getPrimaryKeyField()).getType(), audienceIdString); + getInput.withPrimaryKey(audienceId); + } + + QRecord audienceRecord = new GetAction().executeForRecord(getInput); + if(audienceRecord == null) + { + throw (new QException("A record could not be found for audience type " + audienceType + ", audience id: " + audienceIdString)); + } + } + + //////////////////////////////// + // validate input share scope // + //////////////////////////////// + ShareScope shareScope = SharedRecordProcessUtils.validateScopeId(scopeId); + + /////////////////// + // do the insert // + /////////////////// + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(shareableTableMetaData.getSharedRecordTableName()).withRecord(new QRecord() + .withValue(shareableTableMetaData.getAssetIdFieldName(), recordId) + .withValue(shareableTableMetaData.getScopeFieldName(), shareScope.getPossibleValueId()) + .withValue(shareableAudienceType.getFieldName(), audienceId))); + + ////////////////////// + // check for errors // + ////////////////////// + if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors())) + { + QErrorMessage errorMessage = insertOutput.getRecords().get(0).getErrors().get(0); + if(errorMessage instanceof DuplicateKeyBadInputStatusMessage) + { + throw (new QUserFacingException("This " + assetTableLabel + " has already been shared with this " + audienceTableLabel)); + } + else if(errorMessage instanceof BadInputStatusMessage) + { + throw (new QUserFacingException(errorMessage.getMessage())); + } + throw (new QException("Error sharing " + assetTableLabel + ": " + errorMessage.getMessage())); + } + } + catch(QException qe) + { + throw (qe); + } + catch(Exception e) + { + throw (new QException("Error sharing " + assetTableLabel, e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/ShareScope.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/ShareScope.java new file mode 100644 index 00000000..720402cc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/ShareScope.java @@ -0,0 +1,81 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; + + +/******************************************************************************* + ** for a shared record, what scope of access is given. + *******************************************************************************/ +public enum ShareScope implements PossibleValueEnum +{ + READ_ONLY("Read Only"), + READ_WRITE("Read and Edit"); + + + private final String label; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + ShareScope(String label) + { + this.label = label; + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueId() + { + return name(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return label; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharedRecordProcessUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharedRecordProcessUtils.java new file mode 100644 index 00000000..6cd37a1e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharedRecordProcessUtils.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SharedRecordProcessUtils +{ + /******************************************************************************* + ** + *******************************************************************************/ + record AssetTableAndRecord(QTableMetaData table, ShareableTableMetaData shareableTableMetaData, QRecord record, Serializable recordId) {} + + + + /******************************************************************************* + ** + *******************************************************************************/ + static AssetTableAndRecord getAssetTableAndRecord(String tableName, String recordIdString) throws QException + { + ////////////////////////////// + // validate the asset table // + ////////////////////////////// + QTableMetaData assetTable = QContext.getQInstance().getTable(tableName); + if(assetTable == null) + { + throw (new QException("The specified tableName, " + tableName + ", was not found.")); + } + + ShareableTableMetaData shareableTableMetaData = assetTable.getShareableTableMetaData(); + if(shareableTableMetaData == null) + { + throw (new QException("The specified tableName, " + tableName + ", is not shareable.")); + } + + ////////////////////////////// + // look up the asset record // + ////////////////////////////// + Serializable recordId = ValueUtils.getValueAsFieldType(assetTable.getField(assetTable.getPrimaryKeyField()).getType(), recordIdString); + QRecord assetRecord = new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(recordId)); + if(assetRecord == null) + { + throw (new QException("A record could not be found in table, " + tableName + ", with primary key: " + recordIdString)); + } + + return new AssetTableAndRecord(assetTable, shareableTableMetaData, assetRecord, recordId); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static void assertRecordOwnership(ShareableTableMetaData shareableTableMetaData, QRecord assetRecord, String verbClause) throws QException + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the shareable meta-data says this-table's owner id, then validate that the current user own the record // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(shareableTableMetaData.getThisTableOwnerIdFieldName())) + { + Serializable ownerId = assetRecord.getValue(shareableTableMetaData.getThisTableOwnerIdFieldName()); + if(!Objects.equals(ownerId, QContext.getQSession().getUser().getIdReference())) + { + throw (new QException("You are not the owner of this record, so you may not " + verbClause + " it.")); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static ShareScope validateScopeId(String scopeId) throws QException + { + //////////////////////////////// + // validate input share scope // + //////////////////////////////// + ShareScope shareScope = null; + try + { + shareScope = ShareScope.valueOf(scopeId); + return (shareScope); + } + catch(IllegalArgumentException e) + { + throw (new QException("[" + shareScope + "] is not a recognized value for shareScope. Allowed values are: " + Arrays.toString(ShareScope.values()))); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharingMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharingMetaDataProvider.java new file mode 100644 index 00000000..ffbb13f4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharingMetaDataProvider.java @@ -0,0 +1,61 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SharingMetaDataProvider +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, Consumer processEnricher) throws QException + { + List processes = new ArrayList<>(); + processes.add(new GetSharedRecordsProcess().produce(instance)); + processes.add(new InsertSharedRecordProcess().produce(instance)); + processes.add(new EditSharedRecordProcess().produce(instance)); + processes.add(new DeleteSharedRecordProcess().produce(instance)); + + for(QProcessMetaData process : processes) + { + if(processEnricher != null) + { + processEnricher.accept(process); + } + + instance.addProcess(process); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 55734984..4028c900 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -77,14 +77,14 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine(); private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO) - .withMessageSuffix("because of this process' configuration.") + .withMessageSuffix("because this process is not configured to insert records.") .withSingularFutureMessage("will not be inserted ") .withPluralFutureMessage("will not be inserted ") .withSingularPastMessage("was not inserted ") .withPluralPastMessage("were not inserted "); private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO) - .withMessageSuffix("because of this process' configuration.") + .withMessageSuffix("because this process is not configured to update records.") .withSingularFutureMessage("will not be updated ") .withPluralFutureMessage("will not be updated ") .withSingularPastMessage("was not updated ") @@ -190,6 +190,8 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt this(sourceTable, sourceTableKeyField, destinationTable, destinationTableForeignKey, true, true); } + + /******************************************************************************* ** artificial method, here to make jacoco see that this class is indeed ** included in test coverage... @@ -207,7 +209,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords())) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java index f17b31e0..5a2543b9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep; @@ -248,5 +249,28 @@ public class TableSyncProcess super.withExtractStepClass(extractStepClass); return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLock.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLock.java new file mode 100644 index 00000000..752652bf --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLock.java @@ -0,0 +1,398 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.locks; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** QRecord Entity for ProcessLock table + *******************************************************************************/ +public class ProcessLock extends QRecordEntity +{ + public static final String TABLE_NAME = "processLock"; + + @QField(isEditable = false, isPrimaryKey = true) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String key; + + @QField(possibleValueSourceName = ProcessLockType.TABLE_NAME) + private Integer processLockTypeId; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String userId; + + @QField(label = "Session UUID", maxLength = 36, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String sessionUUID; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String details; + + @QField() + private Instant checkInTimestamp; + + @QField() + private Instant expiresAtTimestamp; + + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public ProcessLock() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public ProcessLock(QRecord record) + { + populateFromQRecord(record); + } + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ProcessLock withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ProcessLock withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ProcessLock withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for key + *******************************************************************************/ + public String getKey() + { + return (this.key); + } + + + + /******************************************************************************* + ** Setter for key + *******************************************************************************/ + public void setKey(String key) + { + this.key = key; + } + + + + /******************************************************************************* + ** Fluent setter for key + *******************************************************************************/ + public ProcessLock withKey(String key) + { + this.key = key; + return (this); + } + + + + /******************************************************************************* + ** Getter for checkInTimestamp + *******************************************************************************/ + public Instant getCheckInTimestamp() + { + return (this.checkInTimestamp); + } + + + + /******************************************************************************* + ** Setter for checkInTimestamp + *******************************************************************************/ + public void setCheckInTimestamp(Instant checkInTimestamp) + { + this.checkInTimestamp = checkInTimestamp; + } + + + + /******************************************************************************* + ** Fluent setter for checkInTimestamp + *******************************************************************************/ + public ProcessLock withCheckInTimestamp(Instant checkInTimestamp) + { + this.checkInTimestamp = checkInTimestamp; + return (this); + } + + + + /******************************************************************************* + ** Getter for expiresAtTimestamp + *******************************************************************************/ + public Instant getExpiresAtTimestamp() + { + return (this.expiresAtTimestamp); + } + + + + /******************************************************************************* + ** Setter for expiresAtTimestamp + *******************************************************************************/ + public void setExpiresAtTimestamp(Instant expiresAtTimestamp) + { + this.expiresAtTimestamp = expiresAtTimestamp; + } + + + + /******************************************************************************* + ** Fluent setter for expiresAtTimestamp + *******************************************************************************/ + public ProcessLock withExpiresAtTimestamp(Instant expiresAtTimestamp) + { + this.expiresAtTimestamp = expiresAtTimestamp; + return (this); + } + + + + /******************************************************************************* + ** Getter for processLockTypeId + *******************************************************************************/ + public Integer getProcessLockTypeId() + { + return (this.processLockTypeId); + } + + + + /******************************************************************************* + ** Setter for processLockTypeId + *******************************************************************************/ + public void setProcessLockTypeId(Integer processLockTypeId) + { + this.processLockTypeId = processLockTypeId; + } + + + + /******************************************************************************* + ** Fluent setter for processLockTypeId + *******************************************************************************/ + public ProcessLock withProcessLockTypeId(Integer processLockTypeId) + { + this.processLockTypeId = processLockTypeId; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public ProcessLock withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for sessionUUID + *******************************************************************************/ + public String getSessionUUID() + { + return (this.sessionUUID); + } + + + + /******************************************************************************* + ** Setter for sessionUUID + *******************************************************************************/ + public void setSessionUUID(String sessionUUID) + { + this.sessionUUID = sessionUUID; + } + + + + /******************************************************************************* + ** Fluent setter for sessionUUID + *******************************************************************************/ + public ProcessLock withSessionUUID(String sessionUUID) + { + this.sessionUUID = sessionUUID; + return (this); + } + + + + /******************************************************************************* + ** Getter for details + *******************************************************************************/ + public String getDetails() + { + return (this.details); + } + + + + /******************************************************************************* + ** Setter for details + *******************************************************************************/ + public void setDetails(String details) + { + this.details = details; + } + + + + /******************************************************************************* + ** Fluent setter for details + *******************************************************************************/ + public ProcessLock withDetails(String details) + { + this.details = details; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java new file mode 100644 index 00000000..b7382f24 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java @@ -0,0 +1,104 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.locks; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; + + +/******************************************************************************* + ** MetaData producer for Process Locks "system" + *******************************************************************************/ +public class ProcessLockMetaDataProducer implements MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException + { + MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput(); + + //////////////////////// + // process lock table // + //////////////////////// + output.add(new QTableMetaData() + .withName(ProcessLock.TABLE_NAME) + .withFieldsFromEntity(ProcessLock.class) + .withIcon(new QIcon().withName("sync_lock")) + .withUniqueKey(new UniqueKey("processLockTypeId", "key")) + .withRecordLabelFormat("%s %s") + .withRecordLabelFields("processLockTypeId", "key") + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "processLockTypeId", "key"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "sessionUUID", "details", "checkInTimestamp", "expiresAtTimestamp"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) + ); + + ///////////////////////////// + // process lock type table // + ///////////////////////////// + output.add(new QTableMetaData() + .withName(ProcessLockType.TABLE_NAME) + .withFieldsFromEntity(ProcessLockType.class) + .withIcon(new QIcon().withName("lock")) + .withUniqueKey(new UniqueKey("name")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "name", "label"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("defaultExpirationSeconds"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) + ); + + /////////////////////////// + // process lock type PVS // + /////////////////////////// + output.add(QPossibleValueSource.newForTable(ProcessLockType.TABLE_NAME)); + + ///////////////////////////////////////////////////// + // join between process lock type and process lock // + ///////////////////////////////////////////////////// + output.add(new QJoinMetaData() + .withLeftTable(ProcessLockType.TABLE_NAME) + .withRightTable(ProcessLock.TABLE_NAME) + .withInferredName() + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("name", "processLockTypeId")) + ); + + return output; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockType.java new file mode 100644 index 00000000..2a9e6209 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockType.java @@ -0,0 +1,262 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.locks; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** QRecord Entity for ProcessLockType table + *******************************************************************************/ +public class ProcessLockType extends QRecordEntity +{ + public static final String TABLE_NAME = "processLockType"; + + @QField(isEditable = false, isPrimaryKey = true) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String name; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String label; + + @QField() + private Integer defaultExpirationSeconds; + + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public ProcessLockType() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public ProcessLockType(QRecord record) + { + populateFromQRecord(record); + } + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ProcessLockType withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ProcessLockType withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ProcessLockType withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public ProcessLockType withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public ProcessLockType withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for defaultExpirationSeconds + *******************************************************************************/ + public Integer getDefaultExpirationSeconds() + { + return (this.defaultExpirationSeconds); + } + + + + /******************************************************************************* + ** Setter for defaultExpirationSeconds + *******************************************************************************/ + public void setDefaultExpirationSeconds(Integer defaultExpirationSeconds) + { + this.defaultExpirationSeconds = defaultExpirationSeconds; + } + + + + /******************************************************************************* + ** Fluent setter for defaultExpirationSeconds + *******************************************************************************/ + public ProcessLockType withDefaultExpirationSeconds(Integer defaultExpirationSeconds) + { + this.defaultExpirationSeconds = defaultExpirationSeconds; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java new file mode 100644 index 00000000..fdc5b20e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java @@ -0,0 +1,486 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.locks; + + +import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility class for working with ProcessLock table - creating, checking-in, + ** and releasing process locks. + *******************************************************************************/ +public class ProcessLockUtils +{ + private static final QLogger LOG = QLogger.getLogger(ProcessLockUtils.class); + + private static Memoization getProcessLockTypeByNameMemoization = new Memoization() + .withTimeout(Duration.ofHours(1)) + .withMayStoreNullValues(false); + + private static Memoization getProcessLockTypeByIdMemoization = new Memoization() + .withTimeout(Duration.ofHours(1)) + .withMayStoreNullValues(false); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessLock create(String key, String typeName, String details) throws UnableToObtainProcessLockException, QException + { + ProcessLockType lockType = getProcessLockTypeByName(typeName); + if(lockType == null) + { + throw (new QException("Unrecognized process lock type: " + typeName)); + } + + QSession qSession = QContext.getQSession(); + + Instant now = Instant.now(); + ProcessLock processLock = new ProcessLock() + .withKey(key) + .withProcessLockTypeId(lockType.getId()) + .withSessionUUID(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUuid(), null)) + .withUserId(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUser().getIdReference(), null)) + .withDetails(details) + .withCheckInTimestamp(now); + + Integer defaultExpirationSeconds = lockType.getDefaultExpirationSeconds(); + if(defaultExpirationSeconds != null) + { + processLock.setExpiresAtTimestamp(now.plusSeconds(defaultExpirationSeconds)); + } + + QRecord insertOutputRecord = tryToInsert(processLock); + + //////////////////////////////////////////////////////////// + // if inserting failed... see if we can get existing lock // + //////////////////////////////////////////////////////////// + StringBuilder existingLockDetails = new StringBuilder(); + ProcessLock existingLock = null; + if(CollectionUtils.nullSafeHasContents(insertOutputRecord.getErrors())) + { + QRecord existingLockRecord = new GetAction().executeForRecord(new GetInput(ProcessLock.TABLE_NAME).withUniqueKey(Map.of("key", key, "processLockTypeId", lockType.getId()))); + if(existingLockRecord != null) + { + existingLock = new ProcessLock(existingLockRecord); + if(StringUtils.hasContent(existingLock.getUserId())) + { + existingLockDetails.append("Held by: ").append(existingLock.getUserId()); + } + + if(StringUtils.hasContent(existingLock.getDetails())) + { + existingLockDetails.append("; with details: ").append(existingLock.getDetails()); + } + + Instant expiresAtTimestamp = existingLock.getExpiresAtTimestamp(); + if(expiresAtTimestamp != null) + { + ZonedDateTime zonedExpiresAt = expiresAtTimestamp.atZone(ValueUtils.getSessionOrInstanceZoneId()); + existingLockDetails.append("; expiring at: ").append(QValueFormatter.formatDateTimeWithZone(zonedExpiresAt)); + } + + if(expiresAtTimestamp != null && expiresAtTimestamp.isBefore(now)) + { + ///////////////////////////////////////////////////////////////////////////////// + // if existing lock has expired, then we can delete it and try to insert again // + ///////////////////////////////////////////////////////////////////////////////// + LOG.info("Existing lock has expired - deleting it and trying again.", logPair("id", existingLock.getId()), + logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", expiresAtTimestamp)); + new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKey(existingLock.getId())); + insertOutputRecord = tryToInsert(processLock); + } + } + else + { + ///////////////////////////////////////////////////////// + // if existing lock doesn't exist, try to insert again // + ///////////////////////////////////////////////////////// + insertOutputRecord = tryToInsert(processLock); + } + } + + if(CollectionUtils.nullSafeHasContents(insertOutputRecord.getErrors())) + { + ///////////////////////////////////////////////////////////////////////////////// + // if at this point, we have errors on the last attempted insert, then give up // + ///////////////////////////////////////////////////////////////////////////////// + LOG.info("Errors in process lock record after attempted insert", logPair("errors", insertOutputRecord.getErrors()), + logPair("key", key), logPair("type", typeName), logPair("details", details)); + throw (new UnableToObtainProcessLockException("A Process Lock already exists for key [" + key + "] of type [" + typeName + "], " + existingLockDetails) + .withExistingLock(existingLock)); + } + + LOG.info("Created process lock", logPair("id", processLock.getId()), + logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", processLock.getExpiresAtTimestamp())); + return new ProcessLock(insertOutputRecord); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QRecord tryToInsert(ProcessLock processLock) throws QException + { + return new InsertAction().execute(new InsertInput(ProcessLock.TABLE_NAME).withRecordEntity(processLock)).getRecords().get(0); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessLock create(String key, String type, String holderId, Duration sleepBetweenTries, Duration maxWait) throws UnableToObtainProcessLockException, QException + { + Instant giveUpTime = Instant.now().plus(maxWait); + + UnableToObtainProcessLockException lastCaughtUnableToObtainProcessLockException = null; + while(true) + { + try + { + ProcessLock processLock = create(key, type, holderId); + return (processLock); + } + catch(UnableToObtainProcessLockException e) + { + lastCaughtUnableToObtainProcessLockException = e; + if(Instant.now().plus(sleepBetweenTries).isBefore(giveUpTime)) + { + SleepUtils.sleep(sleepBetweenTries); + } + else + { + break; + } + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // this variable can never be null with current code-path, but prefer to be defensive regardless // + /////////////////////////////////////////////////////////////////////////////////////////////////// + @SuppressWarnings("ConstantValue") + String suffix = lastCaughtUnableToObtainProcessLockException == null ? "" : ": " + lastCaughtUnableToObtainProcessLockException.getMessage(); + + //noinspection ConstantValue + throw (new UnableToObtainProcessLockException("Unable to obtain process lock for key [" + key + "] in type [" + type + "] after [" + maxWait + "]" + suffix) + .withExistingLock(lastCaughtUnableToObtainProcessLockException == null ? null : lastCaughtUnableToObtainProcessLockException.getExistingLock())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessLock getById(Integer id) throws QException + { + if(id == null) + { + return (null); + } + + QRecord existingLockRecord = new GetAction().executeForRecord(new GetInput(ProcessLock.TABLE_NAME).withPrimaryKey(id)); + if(existingLockRecord != null) + { + return (new ProcessLock(existingLockRecord)); + } + return (null); + } + + + + /******************************************************************************* + ** input wrapper for an overload of the checkin method, to allow more flexibility + ** w/ whether or not you want to update details & expiresAtTimestamp (e.g., so a + ** null can be passed in, to mean "set it to null" vs. "don't update it"). + *******************************************************************************/ + public static class CheckInInput + { + private ProcessLock processLock; + private Instant expiresAtTimestamp = null; + private boolean wasGivenExpiresAtTimestamp = false; + private String details = null; + private boolean wasGivenDetails = false; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public CheckInInput(ProcessLock processLock) + { + this.processLock = processLock; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public CheckInInput withExpiresAtTimestamp(Instant expiresAtTimestamp) + { + this.expiresAtTimestamp = expiresAtTimestamp; + this.wasGivenExpiresAtTimestamp = true; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public CheckInInput withDetails(String details) + { + this.details = details; + this.wasGivenDetails = true; + return (this); + } + } + + + + /******************************************************************************* + ** Do a check-in, with a specific value for the expiresAtTimestamp - which can + ** be set to null to make it null in the lock. + ** + ** If you don't want to specify the expiresAtTimestamp, call the overload that + ** doesn't take the timestamp - in which case it'll either stay the same as it + ** was, or will be set based on the type's default. + *******************************************************************************/ + public static void checkIn(CheckInInput input) + { + ProcessLock processLock = input.processLock; + + try + { + if(processLock == null) + { + LOG.debug("Null processLock passed in - will not checkin."); + return; + } + + QRecord recordToUpdate = new QRecord() + .withValue("id", processLock.getId()) + .withValue("checkInTimestamp", Instant.now()); + + /////////////////////////////////////////////////////////////////// + // if the input was given a details string, update the details // + // use boolean instead of null to know whether or not to do this // + /////////////////////////////////////////////////////////////////// + if(input.wasGivenDetails) + { + recordToUpdate.setValue("details", input.details); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the input object had an expires-at timestamp put in it, then use that value (null or otherwise) for the expires-at-timestamp // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(input.wasGivenExpiresAtTimestamp) + { + recordToUpdate.setValue("expiresAtTimestamp", input.expiresAtTimestamp); + } + else + { + //////////////////////////////////////////////////////////////////////////////// + // else, do the default thing - which is, look for a default in the lock type // + //////////////////////////////////////////////////////////////////////////////// + ProcessLockType lockType = getProcessLockTypeById(processLock.getProcessLockTypeId()); + if(lockType != null) + { + Integer defaultExpirationSeconds = lockType.getDefaultExpirationSeconds(); + if(defaultExpirationSeconds != null) + { + recordToUpdate.setValue("expiresAtTimestamp", Instant.now().plusSeconds(defaultExpirationSeconds)); + } + } + } + + new UpdateAction().execute(new UpdateInput(ProcessLock.TABLE_NAME).withRecord(recordToUpdate)); + LOG.debug("Checked in on process lock", logPair("id", processLock.getId())); + } + catch(Exception e) + { + LOG.warn("Error checking-in on process lock", e, logPair("processLockId", () -> processLock.getId())); + } + } + + + + /******************************************************************************* + ** Do a check-in, with a specific value for the expiresAtTimestamp - which can + ** be set to null to make it null in the lock. + ** + ** If you don't want to specify the expiresAtTimestamp, call the overload that + ** doesn't take the timestamp - in which case it'll either stay the same as it + ** was, or will be set based on the type's default. + *******************************************************************************/ + public static void checkIn(ProcessLock processLock, Instant expiresAtTimestamp) + { + checkIn(new CheckInInput(processLock).withExpiresAtTimestamp(expiresAtTimestamp)); + } + + + + /******************************************************************************* + ** Do a check-in, updating the expires-timestamp based on the lock type's default. + ** (or leaving it the same as it was (null or otherwise) if there is no default + ** on the type). + *******************************************************************************/ + public static void checkIn(ProcessLock processLock) + { + checkIn(new CheckInInput(processLock)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void releaseById(Integer id) + { + if(id == null) + { + LOG.debug("No id passed in to releaseById - returning with noop"); + return; + } + + ProcessLock processLock = null; + try + { + processLock = ProcessLockUtils.getById(id); + if(processLock == null) + { + LOG.info("Process lock not found in releaseById call", logPair("id", id)); + } + } + catch(QException e) + { + LOG.warn("Exception releasing processLock byId", e, logPair("id", id)); + } + + if(processLock != null) + { + release(processLock); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void release(ProcessLock processLock) + { + try + { + if(processLock == null) + { + LOG.debug("No process lock passed in to release - returning with noop"); + return; + } + + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKey(processLock.getId())); + if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors())) + { + throw (new QException("Error deleting processLock record: " + deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString())); + } + } + catch(QException e) + { + LOG.warn("Exception releasing processLock", e, logPair("processLockId", () -> processLock.getId())); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static ProcessLockType getProcessLockTypeByName(String name) + { + Optional result = getProcessLockTypeByNameMemoization.getResult(name, n -> + { + QRecord qRecord = new GetAction().executeForRecord(new GetInput(ProcessLockType.TABLE_NAME).withUniqueKey(Map.of("name", name))); + + if(qRecord != null) + { + return (new ProcessLockType(qRecord)); + } + + return (null); + }); + + return (result.orElse(null)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static ProcessLockType getProcessLockTypeById(Integer id) + { + Optional result = getProcessLockTypeByIdMemoization.getResult(id, i -> + { + QRecord qRecord = new GetAction().executeForRecord(new GetInput(ProcessLockType.TABLE_NAME).withPrimaryKey(id)); + + if(qRecord != null) + { + return (new ProcessLockType(qRecord)); + } + + return (null); + }); + + return (result.orElse(null)); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/UnableToObtainProcessLockException.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/UnableToObtainProcessLockException.java new file mode 100644 index 00000000..c88d58df --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/UnableToObtainProcessLockException.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.locks; + + +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; + + +/******************************************************************************* + ** Lock thrown by ProcessLockUtils when you can't get the lock. + *******************************************************************************/ +public class UnableToObtainProcessLockException extends QUserFacingException +{ + private ProcessLock existingLock; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public UnableToObtainProcessLockException(String message) + { + super(message); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public UnableToObtainProcessLockException(String message, Throwable cause) + { + super(message, cause); + } + + + + /******************************************************************************* + ** Getter for existingLock + *******************************************************************************/ + public ProcessLock getExistingLock() + { + return (this.existingLock); + } + + + + /******************************************************************************* + ** Setter for existingLock + *******************************************************************************/ + public void setExistingLock(ProcessLock existingLock) + { + this.existingLock = existingLock; + } + + + + /******************************************************************************* + ** Fluent setter for existingLock + *******************************************************************************/ + public UnableToObtainProcessLockException withExistingLock(ProcessLock existingLock) + { + this.existingLock = existingLock; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java index a76fe9b0..a3e81f44 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java @@ -343,6 +343,36 @@ public class GeneralProcessUtils + /******************************************************************************* + ** Load rows from a table matching the specified filter, into a map, keyed by the keyFieldName. + ** + ** Note - null values from the key field are NOT put in the map. + ** + ** If multiple values are found for the key, they'll squash each other, and only + ** one (random) value will appear. + *******************************************************************************/ + public static Map loadTableToMap(String tableName, String keyFieldName, Class entityClass, QQueryFilter filter) throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(tableName); + queryInput.setFilter(filter); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + List records = queryOutput.getRecords(); + + Map map = new HashMap<>(); + for(QRecord record : records) + { + Serializable value = record.getValue(keyFieldName); + if(value != null) + { + map.put(value, QRecordEntity.fromQRecord(entityClass, record)); + } + } + return (map); + } + + + /******************************************************************************* ** Load rows from a table matching the specified filter, into a map, keyed by the keyFieldName. ** @@ -412,7 +442,7 @@ public class GeneralProcessUtils *******************************************************************************/ public static Map loadTableToMap(String tableName, String keyFieldName, Class entityClass) throws QException { - return (loadTableToMap(tableName, keyFieldName, entityClass, null)); + return (loadTableToMap(tableName, keyFieldName, entityClass, (Consumer) null)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java new file mode 100644 index 00000000..3dfec3ae --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -0,0 +1,543 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.logging.LogPair; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentityFactory; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableProcessRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableTableAutomationsRunner; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** QQQ service to manage scheduled jobs, using 1 or more Schedulers - implementations + ** of the QSchedulerInterface + *******************************************************************************/ +public class QScheduleManager +{ + private static final QLogger LOG = QLogger.getLogger(QScheduleManager.class); + + private static QScheduleManager qScheduleManager = null; + private final QInstance qInstance; + private final Supplier systemUserSessionSupplier; + + private Map schedulers = new HashMap<>(); + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private QScheduleManager(QInstance qInstance, Supplier systemUserSessionSupplier) + { + this.qInstance = qInstance; + this.systemUserSessionSupplier = systemUserSessionSupplier; + } + + + + /******************************************************************************* + ** Singleton initiator - e.g., must be called to initially initialize the singleton + ** before anyone else calls getInstance (they'll get an error if they call that first). + *******************************************************************************/ + public static QScheduleManager initInstance(QInstance qInstance, Supplier systemUserSessionSupplier) throws QException + { + if(qScheduleManager == null) + { + qScheduleManager = new QScheduleManager(qInstance, systemUserSessionSupplier); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // initialize the scheduler(s) we're configured to use // + // do this, even if we won't start them - so, for example, a web server can still be aware of schedules in the application // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QSchedulerMetaData schedulerMetaData : CollectionUtils.nonNullMap(qInstance.getSchedulers()).values()) + { + QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier); + qScheduleManager.schedulers.put(schedulerMetaData.getName(), scheduler); + } + } + return (qScheduleManager); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void defineDefaultSchedulableTypesInInstance(QInstance qInstance) + { + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.PROCESS.name()).withRunner(new QCodeReference(SchedulableProcessRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.QUEUE_PROCESSOR.name()).withRunner(new QCodeReference(SchedulableSQSQueueRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.TABLE_AUTOMATIONS.name()).withRunner(new QCodeReference(SchedulableTableAutomationsRunner.class))); + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static QScheduleManager getInstance() + { + if(qScheduleManager == null) + { + throw (new IllegalStateException("QScheduleManager singleton has not been init'ed (call initInstance).")); + } + return (qScheduleManager); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void start() throws QException + { + ////////////////////////////////////////////////////////////////////////// + // exit w/o starting schedulers, if schedule manager isn't enabled here // + ////////////////////////////////////////////////////////////////////////// + if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) + { + LOG.info("Not starting ScheduleManager per settings."); + schedulers.values().forEach(s -> s.doNotStart()); + return; + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // ensure that everything which should be scheduled is scheduled, in the appropriate scheduler // + ///////////////////////////////////////////////////////////////////////////////////////////////// + QContext.withTemporaryContext(new CapturedContext(qInstance, systemUserSessionSupplier.get()), () -> setupAllSchedules()); + + ////////////////////////// + // start each scheduler // + ////////////////////////// + schedulers.values().forEach(s -> s.start()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void stop() + { + schedulers.values().forEach(s -> s.stop()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void stopAsync() + { + schedulers.values().forEach(s -> s.stopAsync()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setupAllNewSchedules() throws QException + { + if(QContext.getQInstance().getTables().containsKey(ScheduledJob.TABLE_NAME)) + { + List scheduledJobList = new QueryAction() + .execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true)) + .getRecordEntities(ScheduledJob.class); + + for(ScheduledJob scheduledJob : scheduledJobList) + { + try + { + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + + if(!scheduler.isScheduled(schedulableIdentity, schedulableType)) + { + setupScheduledJob(scheduledJob); + } + } + catch(Exception e) + { + LOG.warn("Error evaluating scheduled job", logPair("id", scheduledJob.getId())); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setupAllSchedules() throws QException + { + ///////////////////////////////////////////// + // read dynamic schedules // + // e.g., user-scheduled processes, reports // + ///////////////////////////////////////////// + List scheduledJobList = null; + try + { + if(QContext.getQInstance().getTables().containsKey(ScheduledJob.TABLE_NAME)) + { + scheduledJobList = new QueryAction() + .execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true)) + .getRecordEntities(ScheduledJob.class); + } + } + catch(Exception e) + { + throw (new QRuntimeException("Failed to query for scheduled jobs - will not set up scheduler!", e)); + } + + ///////////////////////////////////////////////////////// + // let the schedulers know we're starting this process // + ///////////////////////////////////////////////////////// + schedulers.values().forEach(s -> s.startOfSetupSchedules()); + + ///////////////////////// + // schedule all queues // + ///////////////////////// + for(QQueueMetaData queue : qInstance.getQueues().values()) + { + if(queue.getSchedule() != null) + { + setupQueue(queue); + } + } + + //////////////////////////////////////// + // schedule all tables w/ automations // + //////////////////////////////////////// + for(QTableMetaData table : qInstance.getTables().values()) + { + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails != null && automationDetails.getSchedule() != null) + { + setupTableAutomations(table); + } + } + + ///////////////////////////////////////// + // schedule all processes that need it // + ///////////////////////////////////////// + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getSchedule() != null) + { + setupProcess(process); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // todo- before, or after meta-datas? // + // like quartz, it'd just re-schedule if a dupe - but, should we do our own dupe checking? // + ///////////////////////////////////////////////////////////////////////////////////////////// + for(ScheduledJob scheduledJob : CollectionUtils.nonNullList(scheduledJobList)) + { + try + { + setupScheduledJob(scheduledJob); + } + catch(Exception e) + { + LOG.info("Caught exception while scheduling a job", e, logPair("id", scheduledJob.getId())); + } + } + + ////////////////////////////////////////////////////////// + // let the schedulers know we're done with this process // + ////////////////////////////////////////////////////////// + schedulers.values().forEach(s -> s.endOfSetupSchedules()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setupScheduledJob(ScheduledJob scheduledJob) throws QException + { + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + + //////////////////////////////////////////////////////////////////////////////// + // non-active jobs should be deleted from the scheduler. they get re-added // + // if they get re-activated. but we don't want to rely on (e.g., for quartz) // + // the paused state to be drive by is-active. else, devops-pause & unpause // + // operations would clobber scheduled-job record facts // + //////////////////////////////////////////////////////////////////////////////// + if(!scheduledJob.getIsActive()) + { + unscheduleScheduledJob(scheduledJob); + return; + } + + String exceptionSuffix = "in scheduledJob [" + scheduledJob.getId() + "]"; + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // setup schedule meta-data object based on schedule data in the scheduled job - throwing if not well populated // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(scheduledJob.getRepeatSeconds() == null && !StringUtils.hasContent(scheduledJob.getCronExpression())) + { + throw (new QException("Missing a schedule (cronString or repeatSeconds) " + exceptionSuffix)); + } + + QScheduleMetaData scheduleMetaData = new QScheduleMetaData(); + scheduleMetaData.setCronExpression(scheduledJob.getCronExpression()); + scheduleMetaData.setCronTimeZoneId(scheduledJob.getCronTimeZoneId()); + scheduleMetaData.setRepeatSeconds(scheduledJob.getRepeatSeconds()); + + ///////////////////////////////// + // get & validate the job type // + ///////////////////////////////// + if(!StringUtils.hasContent(scheduledJob.getType())) + { + throw (new QException("Missing a type " + exceptionSuffix)); + } + + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + if(schedulableType == null) + { + throw (new QException("Unrecognized type [" + scheduledJob.getType() + "] " + exceptionSuffix)); + } + + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + Map paramMap = new HashMap<>(scheduledJob.getJobParametersMap()); + + SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + runner.validateParams(schedulableIdentity, new HashMap<>(paramMap)); + + scheduler.setupSchedulable(schedulableIdentity, schedulableType, paramMap, scheduleMetaData, true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void unscheduleAll() + { + schedulers.values().forEach(s -> + { + try + { + s.unscheduleAll(); + } + catch(Exception e) + { + LOG.warn("Error unscheduling everything in scheduler " + s, e); + } + }); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void unscheduleScheduledJob(ScheduledJob scheduledJob) throws QException + { + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + + scheduler.unscheduleSchedulable(schedulableIdentity, schedulableType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupProcess(QProcessMetaData process) throws QException + { + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(process); + QSchedulerInterface scheduler = getScheduler(process.getSchedule().getSchedulerName()); + boolean allowedToStart = SchedulerUtils.allowedToStart(process.getName()); + + Map paramMap = new HashMap<>(); + paramMap.put("processName", process.getName()); + + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.name()); + + if(process.getVariantBackend() == null || VariantRunStrategy.SERIAL.equals(process.getVariantRunStrategy())) + { + /////////////////////////////////////////////// + // if no variants, or variant is serial mode // + /////////////////////////////////////////////// + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), process.getSchedule(), allowedToStart); + } + else if(VariantRunStrategy.PARALLEL.equals(process.getVariantRunStrategy())) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // if this a "parallel", which for example means we want to have a thread for each backend variant // + // running at the same time, get the variant records and schedule each separately // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); + for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) + { + try + { + HashMap parameters = new HashMap<>(paramMap); + HashMap variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + parameters.put("backendVariantData", variantMap); + + String identity = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + + BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description); + + scheduler.setupSchedulable(variantIdentity, schedulableType, parameters, process.getSchedule(), allowedToStart); + } + catch(Exception e) + { + LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); + } + } + } + else + { + LOG.error("Unsupported Schedule Run Strategy [" + process.getVariantRunStrategy() + "] was provided."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupTableAutomations(QTableMetaData table) throws QException + { + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.TABLE_AUTOMATIONS.name()); + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + QSchedulerInterface scheduler = getScheduler(automationDetails.getSchedule().getSchedulerName()); + + List tableActionList = PollingAutomationPerTableRunner.getTableActions(qInstance, automationDetails.getProviderName()) + .stream().filter(ta -> ta.tableName().equals(table.getName())) + .toList(); + + for(PollingAutomationPerTableRunner.TableActionsInterface tableActions : tableActionList) + { + SchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(tableActions); + boolean allowedToStart = SchedulerUtils.allowedToStart(table.getName()); + + Map paramMap = new HashMap<>(); + paramMap.put("tableName", tableActions.tableName()); + paramMap.put("automationStatus", tableActions.status().name()); + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), automationDetails.getSchedule(), allowedToStart); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupQueue(QQueueMetaData queue) throws QException + { + SchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(queue); + QSchedulerInterface scheduler = getScheduler(queue.getSchedule().getSchedulerName()); + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.QUEUE_PROCESSOR.name()); + boolean allowedToStart = SchedulerUtils.allowedToStart(queue.getName()); + + Map paramMap = new HashMap<>(); + paramMap.put("queueName", queue.getName()); + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), queue.getSchedule(), allowedToStart); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QSchedulerInterface getScheduler(String schedulerName) throws QException + { + if(!StringUtils.hasContent(schedulerName)) + { + throw (new QException("Scheduler name was not given (and the concept of a default scheduler does not exist at this time).")); + } + + QSchedulerInterface scheduler = schedulers.get(schedulerName); + if(scheduler == null) + { + throw (new QException("Unrecognized schedulerName [" + schedulerName + "]")); + } + + return (scheduler); + } + + + + /******************************************************************************* + ** reset the singleton instance (to null); clear the map of schedulers. + ** Not clear it's ever useful to call in main-code - but can be used for tests. + *******************************************************************************/ + public void unInit() + { + qScheduleManager = null; + schedulers.values().forEach(s -> s.unInit()); + schedulers.clear(); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java new file mode 100644 index 00000000..3d99ee0b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface QSchedulerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + String getSchedulerName(); + + /******************************************************************************* + ** + *******************************************************************************/ + void start(); + + /******************************************************************************* + ** called to indicate that the schedule manager is past its startup routine, + ** but that the schedule should not actually be running in this process. + *******************************************************************************/ + default void doNotStart() + { + + } + + /******************************************************************************* + ** + *******************************************************************************/ + void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType); + + /******************************************************************************* + ** + *******************************************************************************/ + boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType); + + /******************************************************************************* + ** + *******************************************************************************/ + void unscheduleAll() throws QException; + + /******************************************************************************* + ** + *******************************************************************************/ + void stopAsync(); + + /******************************************************************************* + ** + *******************************************************************************/ + void stop(); + + /******************************************************************************* + ** Handle a whole shutdown of the scheduler system (e.g., between unit tests). + *******************************************************************************/ + default void unInit() + { + ///////////////////// + // noop by default // + ///////////////////// + } + + /******************************************************************************* + ** let the scheduler know when the schedule manager is at the start of setting up schedules. + *******************************************************************************/ + default void startOfSetupSchedules() + { + ///////////////////// + // noop by default // + ///////////////////// + } + + /******************************************************************************* + ** let the scheduler know when the schedule manager is at the end of setting up schedules. + *******************************************************************************/ + default void endOfSetupSchedules() + { + ///////////////////// + // noop by default // + ///////////////////// + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java deleted file mode 100644 index 375aebb9..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java +++ /dev/null @@ -1,502 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.scheduler; - - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; -import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; -import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; -import com.kingsrook.qqq.backend.core.logging.LogPair; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; -import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; - - -/******************************************************************************* - ** QQQ Service (Singleton) that starts up repeating, scheduled jobs within QQQ. - ** - ** These include: - ** - Automation providers (which require polling) - ** - Queue pollers - ** - Scheduled processes. - ** - ** All of these jobs run using a "system session" - as defined by the sessionSupplier. - *******************************************************************************/ -public class ScheduleManager -{ - private static final QLogger LOG = QLogger.getLogger(ScheduleManager.class); - - private static ScheduleManager scheduleManager = null; - private final QInstance qInstance; - - protected Supplier sessionSupplier; - - ///////////////////////////////////////////////////////////////////////////////////// - // for jobs that don't define a delay index, auto-stagger them, using this counter // - ///////////////////////////////////////////////////////////////////////////////////// - private int delayIndex = 0; - - private List executors = new ArrayList<>(); - - - - /******************************************************************************* - ** Singleton constructor - *******************************************************************************/ - private ScheduleManager(QInstance qInstance) - { - this.qInstance = qInstance; - } - - - - /******************************************************************************* - ** Singleton accessor - *******************************************************************************/ - public static ScheduleManager getInstance(QInstance qInstance) - { - if(scheduleManager == null) - { - scheduleManager = new ScheduleManager(qInstance); - } - return (scheduleManager); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public void start() - { - if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) - { - LOG.info("Not starting ScheduleManager per settings."); - return; - } - - boolean needToClearContext = false; - try - { - if(QContext.getQInstance() == null) - { - needToClearContext = true; - QContext.init(qInstance, sessionSupplier.get()); - } - - for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values()) - { - startQueueProvider(queueProvider); - } - - for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values()) - { - startAutomationProviderPerTable(automationProvider); - } - - for(QProcessMetaData process : qInstance.getProcesses().values()) - { - if(process.getSchedule() != null && allowedToStart(process.getName())) - { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - /////////////////////////////////////////////// - // if no variants, or variant is serial mode // - /////////////////////////////////////////////// - startProcess(process, null); - } - else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) - { - ///////////////////////////////////////////////////////////////////////////////////////////////////// - // if this a "parallel", which for example means we want to have a thread for each backend variant // - // running at the same time, get the variant records and schedule each separately // - ///////////////////////////////////////////////////////////////////////////////////////////////////// - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - for(QRecord qRecord : CollectionUtils.nonNullList(getBackendVariantFilteredRecords(process))) - { - try - { - startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); - } - catch(Exception e) - { - LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); - } - } - } - else - { - LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); - } - } - } - } - finally - { - if(needToClearContext) - { - QContext.clear(); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private List getBackendVariantFilteredRecords(QProcessMetaData processMetaData) - { - List records = null; - try - { - QScheduleMetaData scheduleMetaData = processMetaData.getSchedule(); - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); - - QContext.init(qInstance, sessionSupplier.get()); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - records = queryOutput.getRecords(); - } - catch(Exception e) - { - LOG.error("An error fetching variant data for process [" + processMetaData.getLabel() + "]", e); - } - - return (records); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startAutomationProviderPerTable(QAutomationProviderMetaData automationProvider) - { - /////////////////////////////////////////////////////////////////////////////////// - // ask the PollingAutomationPerTableRunner how many threads of itself need setup // - // then start a scheduled executor foreach one // - /////////////////////////////////////////////////////////////////////////////////// - List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); - for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions) - { - if(allowedToStart(tableAction.tableName())) - { - PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableAction); - StandardScheduledExecutor executor = new StandardScheduledExecutor(runner); - - QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule); - - executor.setName(runner.getName()); - setScheduleInExecutor(schedule, executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - - executors.add(executor); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private boolean allowedToStart(String name) - { - String propertyName = "qqq.scheduleManager.onlyStartNamesMatching"; - String propertyValue = System.getProperty(propertyName, ""); - if(propertyValue.equals("")) - { - return (true); - } - - return (name.matches(propertyValue)); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startQueueProvider(QQueueProviderMetaData queueProvider) - { - if(allowedToStart(queueProvider.getName())) - { - switch(queueProvider.getType()) - { - case SQS: - startSqsProvider((SQSQueueProviderMetaData) queueProvider); - break; - default: - throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startSqsProvider(SQSQueueProviderMetaData queueProvider) - { - QInstance scheduleManagerQueueInstance = qInstance; - Supplier scheduleManagerSessionSupplier = sessionSupplier; - - for(QQueueMetaData queue : qInstance.getQueues().values()) - { - if(queueProvider.getName().equals(queue.getProviderName()) && allowedToStart(queue.getName())) - { - SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); - sqsQueuePoller.setQueueProviderMetaData(queueProvider); - sqsQueuePoller.setQueueMetaData(queue); - sqsQueuePoller.setQInstance(scheduleManagerQueueInstance); - sqsQueuePoller.setSessionSupplier(scheduleManagerSessionSupplier); - - StandardScheduledExecutor executor = new StandardScheduledExecutor(sqsQueuePoller); - - QScheduleMetaData schedule = Objects.requireNonNullElseGet(queue.getSchedule(), - () -> Objects.requireNonNullElseGet(queueProvider.getSchedule(), - this::getDefaultSchedule)); - - executor.setName(queue.getName()); - setScheduleInExecutor(schedule, executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - - executors.add(executor); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startProcess(QProcessMetaData process, Map backendVariantData) - { - Runnable runProcess = () -> - { - String originalThreadName = Thread.currentThread().getName(); - - try - { - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) - { - QContext.init(qInstance, sessionSupplier.get()); - executeSingleProcess(process, backendVariantData); - } - else if(QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - /////////////////////////////////////////////////////////////////////////////////////////////////// - // if this is "serial", which for example means we want to run each backend variant one after // - // the other in the same thread so loop over these here so that they run in same lambda function // - /////////////////////////////////////////////////////////////////////////////////////////////////// - for(QRecord qRecord : getBackendVariantFilteredRecords(process)) - { - try - { - QContext.init(qInstance, sessionSupplier.get()); - QScheduleMetaData scheduleMetaData = process.getSchedule(); - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - executeSingleProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); - } - catch(Exception e) - { - LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); - } - } - } - } - catch(Exception e) - { - LOG.warn("Exception thrown running scheduled process [" + process.getName() + "]", e); - } - finally - { - Thread.currentThread().setName(originalThreadName); - QContext.clear(); - } - }; - - StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess); - executor.setName("process:" + process.getName()); - setScheduleInExecutor(process.getSchedule(), executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - - executors.add(executor); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static void executeSingleProcess(QProcessMetaData process, Map backendVariantData) throws QException - { - if(backendVariantData != null) - { - QContext.getQSession().setBackendVariants(backendVariantData); - } - - Thread.currentThread().setName("ScheduledProcess>" + process.getName()); - LOG.debug("Running Scheduled Process [" + process.getName() + "]"); - - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(process.getName()); - runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - - QContext.pushAction(runProcessInput); - - RunProcessAction runProcessAction = new RunProcessAction(); - runProcessAction.execute(runProcessInput); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void setScheduleInExecutor(QScheduleMetaData schedule, StandardScheduledExecutor executor) - { - if(schedule.getRepeatMillis() != null) - { - executor.setDelayMillis(schedule.getRepeatMillis()); - } - else - { - executor.setDelayMillis(1000 * schedule.getRepeatSeconds()); - } - - if(schedule.getInitialDelayMillis() != null) - { - executor.setInitialDelayMillis(schedule.getInitialDelayMillis()); - } - else if(schedule.getInitialDelaySeconds() != null) - { - executor.setInitialDelayMillis(1000 * schedule.getInitialDelaySeconds()); - } - else - { - executor.setInitialDelayMillis(1000 * ++delayIndex); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private QScheduleMetaData getDefaultSchedule() - { - QScheduleMetaData schedule; - schedule = new QScheduleMetaData() - .withInitialDelaySeconds(delayIndex++) - .withRepeatSeconds(60); - return schedule; - } - - - - /******************************************************************************* - ** Setter for sessionSupplier - ** - *******************************************************************************/ - public void setSessionSupplier(Supplier sessionSupplier) - { - this.sessionSupplier = sessionSupplier; - } - - - - /******************************************************************************* - ** Getter for managedExecutors - ** - *******************************************************************************/ - public List getExecutors() - { - return executors; - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public void stopAsync() - { - for(StandardScheduledExecutor scheduledExecutor : executors) - { - scheduledExecutor.stopAsync(); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - static void resetSingleton() - { - scheduleManager = null; - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java new file mode 100644 index 00000000..c7cc6a4c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -0,0 +1,198 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.LogPair; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; + + +/******************************************************************************* + ** Utility methods used by various schedulers. + *******************************************************************************/ +public class SchedulerUtils +{ + private static final QLogger LOG = QLogger.getLogger(SchedulerUtils.class); + + + /******************************************************************************* + ** + *******************************************************************************/ + public static boolean allowedToStart(String name) + { + String propertyName = "qqq.scheduleManager.onlyStartNamesMatching"; + String propertyValue = System.getProperty(propertyName, ""); + if(propertyValue.equals("")) + { + return (true); + } + + return (name.matches(propertyValue)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void runProcess(QInstance qInstance, Supplier sessionSupplier, QProcessMetaData process, Map backendVariantData, Map processInputValues) + { + String originalThreadName = Thread.currentThread().getName(); + + try + { + QContext.init(qInstance, sessionSupplier.get()); + + if(process.getVariantBackend() == null || VariantRunStrategy.PARALLEL.equals(process.getVariantRunStrategy())) + { + executeSingleProcess(process, backendVariantData, processInputValues); + } + else if(VariantRunStrategy.SERIAL.equals(process.getVariantRunStrategy())) + { + /////////////////////////////////////////////////////////////////////////////////////////////////// + // if this is "serial", which for example means we want to run each backend variant one after // + // the other in the same thread so loop over these here so that they run in same lambda function // + /////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord qRecord : getBackendVariantFilteredRecords(process)) + { + try + { + QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); + Map thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())); + executeSingleProcess(process, thisVariantData, processInputValues); + } + catch(Exception e) + { + LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); + } + } + } + } + catch(Exception e) + { + LOG.warn("Exception thrown running scheduled process [" + process.getName() + "]", e); + } + finally + { + Thread.currentThread().setName(originalThreadName); + QContext.clear(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void executeSingleProcess(QProcessMetaData process, Map backendVariantData, Map processInputValues) throws QException + { + if(backendVariantData != null) + { + QContext.getQSession().setBackendVariants(backendVariantData); + } + + Thread.currentThread().setName("ScheduledProcess>" + process.getName()); + LOG.debug("Running Scheduled Process [" + process.getName() + "] with values [" + processInputValues + "]"); + + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(process.getName()); + + Serializable recordId = null; + for(Map.Entry entry : CollectionUtils.nonNullMap(processInputValues).entrySet()) + { + runProcessInput.withValue(entry.getKey(), entry.getValue()); + if(entry.getKey().equals("recordId")) + { + recordId = entry.getValue(); + } + } + + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there was a "recordId" input value, and this table is for a process, then set up a callback to get the record // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(recordId != null && StringUtils.hasContent(process.getTableName())) + { + QTableMetaData table = QContext.getQInstance().getTable(process.getTableName()); + runProcessInput.setCallback(QProcessCallbackFactory.forPrimaryKey(table.getPrimaryKeyField(), recordId)); + } + + QContext.pushAction(runProcessInput); + + RunProcessAction runProcessAction = new RunProcessAction(); + runProcessAction.execute(runProcessInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List getBackendVariantFilteredRecords(QProcessMetaData processMetaData) + { + List records = null; + try + { + QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(processMetaData.getVariantBackend()); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + records = queryOutput.getRecords(); + } + catch(Exception e) + { + LOG.error("An error fetching variant data for process [" + processMetaData.getLabel() + "]", e); + } + + return (records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java new file mode 100644 index 00000000..9c3840ca --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to reschedule all scheduled jobs (in all schedulers). + *******************************************************************************/ +public class RescheduleAllJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Reschedule all Scheduled Jobs") + .withIcon(new QIcon("update")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to reschedule all jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All jobs have been rescheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().setupAllSchedules(); + } + catch(Exception e) + { + throw (new QException("Error setting up all scheduled jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java new file mode 100644 index 00000000..408ec1b8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to schedule all new scheduled jobs (in all schedulers). + *******************************************************************************/ +public class ScheduleAllNewJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Schedule all New Scheduled Jobs") + .withIcon(new QIcon("more_time")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to schedule all new jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All new jobs have been scheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().setupAllNewSchedules(); + } + catch(Exception e) + { + throw (new QException("Error scheduling new jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java new file mode 100644 index 00000000..11492b06 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to unschedule all scheduled jobs (in all schedulers). + *******************************************************************************/ +public class UnscheduleAllJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Unschedule all Scheduled Jobs") + .withIcon(new QIcon("update_disabled")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to unschedule all jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All jobs have been unscheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().unscheduleAll(); + } + catch(Exception e) + { + throw (new QException("Error unscheduling all scheduled jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java new file mode 100644 index 00000000..62ec6cb7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import org.quartz.JobDetail; +import org.quartz.Trigger; + + +/******************************************************************************* + ** + *******************************************************************************/ +public record QuartzJobAndTriggerWrapper(JobDetail jobDetail, Trigger trigger, Trigger.TriggerState triggerState) +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java new file mode 100644 index 00000000..f635bd9b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java @@ -0,0 +1,109 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import org.apache.logging.log4j.Level; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +@DisallowConcurrentExecution +public class QuartzJobRunner implements Job +{ + private static final QLogger LOG = QLogger.getLogger(QuartzJobRunner.class); + + private static Level logLevel = null; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void execute(JobExecutionContext context) throws JobExecutionException + { + CapturedContext capturedContext = QContext.capture(); + + String name = null; + SchedulableType schedulableType = null; + Map params = null; + try + { + name = context.getJobDetail().getKey().getName(); + + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); + QInstance qInstance = quartzScheduler.getQInstance(); + QContext.init(qInstance, quartzScheduler.getSessionSupplier().get()); + + schedulableType = qInstance.getSchedulableType(context.getJobDetail().getJobDataMap().getString("type")); + params = (Map) context.getJobDetail().getJobDataMap().get("params"); + + SchedulableRunner schedulableRunner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + + if(logLevel != null) + { + LOG.log(logLevel, "Running QuartzJob", null, logPair("name", name), logPair("type", schedulableType.getName()), logPair("params", params)); + } + + schedulableRunner.run(params); + + if(logLevel != null) + { + LOG.log(logLevel, "Finished QuartzJob", null, logPair("name", name), logPair("type", schedulableType.getName()), logPair("params", params)); + } + } + catch(Exception e) + { + LOG.warn("Error running QuartzJob", e, logPair("name", name), logPair("type", schedulableType == null ? null : schedulableType.getName()), logPair("params", params)); + } + finally + { + QContext.init(capturedContext); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void setLogLevel(Level level) + { + logLevel = level; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java new file mode 100644 index 00000000..a9bb5434 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -0,0 +1,763 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.io.Serializable; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.TimeZone; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import org.quartz.CronExpression; +import org.quartz.CronScheduleBuilder; +import org.quartz.Job; +import org.quartz.JobBuilder; +import org.quartz.JobDetail; +import org.quartz.JobKey; +import org.quartz.ScheduleBuilder; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.SimpleScheduleBuilder; +import org.quartz.Trigger; +import org.quartz.TriggerBuilder; +import org.quartz.TriggerKey; +import org.quartz.impl.StdSchedulerFactory; +import org.quartz.impl.matchers.GroupMatcher; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Singleton to provide access between QQQ and the quartz Scheduler system. + *******************************************************************************/ +public class QuartzScheduler implements QSchedulerInterface +{ + private static final QLogger LOG = QLogger.getLogger(QuartzScheduler.class); + + private static QuartzScheduler quartzScheduler = null; + + private final QInstance qInstance; + private String schedulerName; + private Supplier sessionSupplier; + + private Scheduler scheduler; + + + ///////////////////////////////////////////////////////////////////////////////////////// + // create memoization objects for some quartz-query functions, that we'll only want to // + // use during our setup routine, when we'd query it many times over and over again. // + // So default to a timeout of 0 (effectively disabling memoization). then in the // + // start-of-setup and end-of-setup methods, temporarily increase, then re-decrease // + ///////////////////////////////////////////////////////////////////////////////////////// + private Memoization> jobGroupNamesMemoization = new Memoization>() + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + + private Memoization> jobKeyNamesMemoization = new Memoization>() + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + + private Memoization> queryQuartzMemoization = new Memoization>() + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + + private List> allMemoizations = List.of(jobGroupNamesMemoization, jobKeyNamesMemoization, queryQuartzMemoization); + + /////////////////////////////////////////////////////////////////////////////// + // vars used during the setup routine, to figure out what jobs need deleted. // + /////////////////////////////////////////////////////////////////////////////// + private boolean insideSetup = false; + private List scheduledJobsAtStartOfSetup = new ArrayList<>(); + private List scheduledJobsAtEndOfSetup = new ArrayList<>(); + + ///////////////////////////////////////////////////////////////////////////////// + // track if the instance is past the server's startup routine. // + // for quartz - we'll use this to know if we're allowed to schedule jobs. // + // that is - during server startup, we don't want to the schedule & unschedule // + // routine, which could potentially have serve concurrency problems // + ///////////////////////////////////////////////////////////////////////////////// + private boolean pastStartup = false; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + private QuartzScheduler(QInstance qInstance, String schedulerName, Supplier sessionSupplier) + { + this.qInstance = qInstance; + this.schedulerName = schedulerName; + this.sessionSupplier = sessionSupplier; + } + + + + /******************************************************************************* + ** Singleton initiator - e.g., must be called to initially initialize the singleton + ** before anyone else calls getInstance (they'll get an error if they call that first). + *******************************************************************************/ + public static QuartzScheduler initInstance(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier sessionSupplier) throws SchedulerException + { + if(quartzScheduler == null) + { + quartzScheduler = new QuartzScheduler(qInstance, schedulerName, sessionSupplier); + + /////////////////////////////////////////////////////////// + // Grab the Scheduler instance from the Factory // + // initialize it with the properties we took in as input // + /////////////////////////////////////////////////////////// + StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); + schedulerFactory.initialize(quartzProperties); + quartzScheduler.scheduler = schedulerFactory.getScheduler(); + } + return (quartzScheduler); + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static QuartzScheduler getInstance() + { + if(quartzScheduler == null) + { + throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed (call initInstance).")); + } + return (quartzScheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getSchedulerName() + { + return (schedulerName); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void start() + { + this.pastStartup = true; + + try + { + ////////////////////// + // and start it off // + ////////////////////// + scheduler.start(); + } + catch(Exception e) + { + LOG.error("Error starting quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void doNotStart() + { + this.pastStartup = true; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stop() + { + try + { + scheduler.shutdown(true); + } + catch(SchedulerException e) + { + LOG.error("Error shutting down (stopping) quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stopAsync() + { + try + { + scheduler.shutdown(false); + } + catch(SchedulerException e) + { + LOG.error("Error shutting down (stopping) quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart) + { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + + Map jobData = new HashMap<>(); + jobData.put("params", parameters); + jobData.put("type", schedulableType.getName()); + + scheduleJob(schedulableIdentity, schedulableType.getName(), QuartzJobRunner.class, jobData, schedule, allowedToStart); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void startOfSetupSchedules() + { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + + this.insideSetup = true; + this.allMemoizations.forEach(m -> m.setTimeout(Duration.ofSeconds(5))); + + try + { + this.scheduledJobsAtStartOfSetup = queryQuartz(); + this.scheduledJobsAtEndOfSetup = new ArrayList<>(); + } + catch(Exception e) + { + LOG.warn("Error querying quartz for the currently scheduled jobs during startup - will not be able to delete no-longer-needed jobs!", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void endOfSetupSchedules() + { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + + this.insideSetup = false; + this.allMemoizations.forEach(m -> m.setTimeout(Duration.ofSeconds(0))); + + if(this.scheduledJobsAtStartOfSetup == null) + { + return; + } + + try + { + Set startJobKeys = this.scheduledJobsAtStartOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); + Set endJobKeys = this.scheduledJobsAtEndOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // remove all 'end' keys from the set of start keys. any left-over start-keys need to be deleted. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + startJobKeys.removeAll(endJobKeys); + for(JobKey jobKey : startJobKeys) + { + LOG.info("Deleting job that had previously been scheduled, but doesn't appear to be any more", logPair("jobKey", jobKey)); + deleteJob(jobKey); + } + } + catch(Exception e) + { + LOG.warn("Error trying to clean up no-longer-needed jobs at end of scheduler setup", e); + } + + //////////////////////////////////////////////////// + // reset these lists, no need to keep them around // + //////////////////////////////////////////////////// + this.scheduledJobsAtStartOfSetup = null; + this.scheduledJobsAtEndOfSetup = null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean scheduleJob(SchedulableIdentity schedulableIdentity, String groupName, Class jobClass, Map jobData, QScheduleMetaData scheduleMetaData, boolean allowedToStart) + { + try + { + ///////////////////////// + // Define job instance // + ///////////////////////// + JobKey jobKey = new JobKey(schedulableIdentity.getIdentity(), groupName); + JobDetail jobDetail = JobBuilder.newJob(jobClass) + .withIdentity(jobKey) + .withDescription(schedulableIdentity.getDescription()) + .storeDurably() + .requestRecovery() // todo - our frequent repeaters, maybe nice to say false here + .build(); + + jobDetail.getJobDataMap().putAll(jobData); + + ///////////////////////////////////////////////////////// + // map the qqq schedule meta data to a quartz schedule // + ///////////////////////////////////////////////////////// + ScheduleBuilder scheduleBuilder; + if(scheduleMetaData.isCron()) + { + CronExpression cronExpression = new CronExpression(scheduleMetaData.getCronExpression()); + cronExpression.setTimeZone(TimeZone.getTimeZone(scheduleMetaData.getCronTimeZoneId())); + scheduleBuilder = CronScheduleBuilder.cronSchedule(cronExpression); + } + else + { + long intervalMillis = Objects.requireNonNullElseGet(scheduleMetaData.getRepeatMillis(), () -> scheduleMetaData.getRepeatSeconds() * 1000); + scheduleBuilder = SimpleScheduleBuilder.simpleSchedule() + .withIntervalInMilliseconds(intervalMillis) + .repeatForever(); + } + + Date startAt = new Date(); + if(scheduleMetaData.getInitialDelayMillis() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis()); + } + else if(scheduleMetaData.getInitialDelaySeconds() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000); + } + else + { + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // by default, put a 3-second delay on everything we schedule // + // this gives us a chance to re-pause if the job was previously paused, but then we re-schedule it. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + startAt.setTime(startAt.getTime() + 3000); + } + + /////////////////////////////////////// + // Define a Trigger for the schedule // + /////////////////////////////////////// + Trigger trigger = TriggerBuilder.newTrigger() + .withIdentity(new TriggerKey(schedulableIdentity.getIdentity(), groupName)) + .withDescription(schedulableIdentity.getDescription() + " - " + getScheduleDescriptionForTrigger(scheduleMetaData)) + .forJob(jobKey) + .withSchedule(scheduleBuilder) + // .startAt(startAt) + .build(); + + /////////////////////////////////////// + // Schedule the job with the trigger // + /////////////////////////////////////// + addOrReplaceJobAndTrigger(jobKey, jobDetail, trigger); + + /////////////////////////////////////////////////////////////////////////// + // if we're inside the setup event (e.g., initial startup), then capture // + // this job as one that is currently active and should be kept. // + /////////////////////////////////////////////////////////////////////////// + if(insideSetup) + { + scheduledJobsAtEndOfSetup.add(new QuartzJobAndTriggerWrapper(jobDetail, trigger, null)); + } + + return (true); + } + catch(Exception e) + { + LOG.warn("Error scheduling job", e, logPair("name", schedulableIdentity.getIdentity()), logPair("group", groupName)); + return (false); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getScheduleDescriptionForTrigger(QScheduleMetaData scheduleMetaData) + { + if(StringUtils.hasContent(scheduleMetaData.getDescription())) + { + return scheduleMetaData.getDescription(); + } + + if(StringUtils.hasContent(scheduleMetaData.getCronExpression())) + { + return "cron expression: " + scheduleMetaData.getCronExpression() + (StringUtils.hasContent(scheduleMetaData.getCronTimeZoneId()) ? " time zone: " + scheduleMetaData.getCronTimeZoneId() : ""); + } + + if(scheduleMetaData.getRepeatSeconds() != null) + { + return "repeat seconds: " + scheduleMetaData.getRepeatSeconds(); + } + + return ""; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + + deleteJob(new JobKey(schedulableIdentity.getIdentity(), schedulableType.getName())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + try + { + JobKey jobKey = new JobKey(schedulableIdentity.getIdentity(), schedulableType.getName()); + return (isJobAlreadyScheduled(jobKey)); + } + catch(Exception e) + { + LOG.warn("Error checking if job is scheduled", logPair("identity", schedulableIdentity)); + return (false); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleAll() throws QException + { + try + { + for(QuartzJobAndTriggerWrapper wrapper : queryQuartz()) + { + deleteJob(new JobKey(wrapper.jobDetail().getKey().getName(), wrapper.jobDetail().getKey().getGroup())); + } + } + catch(Exception e) + { + throw (new QException("Error unscheduling all quartz jobs", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addOrReplaceJobAndTrigger(JobKey jobKey, JobDetail jobDetail, Trigger trigger) throws SchedulerException + { + boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey); + if(isJobAlreadyScheduled) + { + boolean wasPaused = wasExistingJobPaused(jobKey); + + this.scheduler.scheduleJob(jobDetail, Set.of(trigger), true); // note, true flag here replaces if already present. + LOG.info("Re-scheduled job", logPair("jobKey", jobKey)); + + if(wasPaused) + { + LOG.info("Re-pausing job", logPair("jobKey", jobKey)); + pauseJob(jobKey.getName(), jobKey.getGroup()); + } + } + else + { + this.scheduler.scheduleJob(jobDetail, trigger); + LOG.info("Scheduled new job", logPair("jobKey", jobKey)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean wasExistingJobPaused(JobKey jobKey) throws SchedulerException + { + List quartzJobAndTriggerWrappers = queryQuartz(); + Optional existingWrapper = quartzJobAndTriggerWrappers.stream().filter(w -> w.jobDetail().getKey().equals(jobKey)).findFirst(); + if(existingWrapper.isPresent()) + { + if(Trigger.TriggerState.PAUSED.equals(existingWrapper.get().triggerState())) + { + return (true); + } + } + + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean isJobAlreadyScheduled(JobKey jobKey) throws SchedulerException + { + Optional> jobGroupNames = jobGroupNamesMemoization.getResult(AnyKey.getInstance(), (x) -> scheduler.getJobGroupNames()); + if(jobGroupNames.isEmpty()) + { + throw (new SchedulerException("Error getting job group names")); + } + + for(String group : jobGroupNames.get()) + { + Optional> jobKeys = jobKeyNamesMemoization.getResult(group, (x) -> scheduler.getJobKeys(GroupMatcher.groupEquals(group))); + if(jobKeys.isEmpty()) + { + throw (new SchedulerException("Error getting job keys")); + } + + if(jobKeys.get().contains(jobKey)) + { + return (true); + } + } + + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean deleteJob(JobKey jobKey) + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html // + // Deleting a Job and Unscheduling All of Its Triggers // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(isJobAlreadyScheduled(jobKey)) + { + boolean result = scheduler.deleteJob(jobKey); + LOG.info("Attempted to delete quartz job", logPair("jobKey", jobKey), logPair("deleteJobResult", result)); + return (result); + } + + ///////////////////////////////////////// + // return true to indicate, we're good // + ///////////////////////////////////////// + LOG.info("Request to delete quartz job, but it is not already scheduled.", logPair("jobKey", jobKey)); + return (true); + } + catch(Exception e) + { + LOG.warn("Error deleting job", e, logPair("jobKey", jobKey)); + return false; + } + } + + + + /******************************************************************************* + ** Getter for qInstance + ** + *******************************************************************************/ + public QInstance getQInstance() + { + return qInstance; + } + + + + /******************************************************************************* + ** Getter for sessionSupplier + ** + *******************************************************************************/ + public Supplier getSessionSupplier() + { + return sessionSupplier; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void pauseAll() throws SchedulerException + { + /////////////////////////////////////////////////////////////////////////////// + // lesson from past self to future self: // + // pauseAll creates paused-group entries for all jobs - // + // and so they can only really be resumed by a resumeAll call... // + // even newly scheduled things become paused. Which can be quite confusing. // + // so, we don't want pause all. // + /////////////////////////////////////////////////////////////////////////////// + // this.scheduler.pauseAll(); + + List quartzJobAndTriggerWrappers = queryQuartz(); + for(QuartzJobAndTriggerWrapper wrapper : quartzJobAndTriggerWrappers) + { + this.pauseJob(wrapper.jobDetail().getKey().getName(), wrapper.jobDetail().getKey().getGroup()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void resumeAll() throws SchedulerException + { + ////////////////////////////////////////////////// + // this seems okay, even though pauseAll isn't. // + ////////////////////////////////////////////////// + this.scheduler.resumeAll(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void pauseJob(String jobName, String groupName) throws SchedulerException + { + LOG.info("Request to pause job", logPair("jobName", jobName)); + this.scheduler.pauseJob(new JobKey(jobName, groupName)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void resumeJob(String jobName, String groupName) throws SchedulerException + { + LOG.info("Request to resume job", logPair("jobName", jobName)); + this.scheduler.resumeJob(new JobKey(jobName, groupName)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public List queryQuartz() throws SchedulerException + { + return queryQuartzMemoization.getResultThrowing(AnyKey.getInstance(), (x) -> + { + List rs = new ArrayList<>(); + + for(String group : scheduler.getJobGroupNames()) + { + Set jobKeys = scheduler.getJobKeys(GroupMatcher.groupEquals(group)); + for(JobKey jobKey : jobKeys) + { + JobDetail jobDetail = scheduler.getJobDetail(jobKey); + List triggersOfJob = scheduler.getTriggersOfJob(jobKey); + for(Trigger trigger : triggersOfJob) + { + Trigger.TriggerState triggerState = scheduler.getTriggerState(trigger.getKey()); + rs.add(new QuartzJobAndTriggerWrapper(jobDetail, trigger, triggerState)); + } + } + } + + return (rs); + }).orElse(null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unInit() + { + /////////////////////////////////////////////////// + // resetting the singleton should be sufficient! // + /////////////////////////////////////////////////// + quartzScheduler = null; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java new file mode 100644 index 00000000..123b48c9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** Manage process to pause all quartz jobs + *******************************************************************************/ +public class PauseAllQuartzJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Pause All Quartz Jobs") + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to pause all quartz jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All quartz jobs have been paused"))))) + .withIcon(new QIcon("pause_circle_outline")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler.getInstance().pauseAll(); + } + catch(Exception e) + { + throw (new QException("Error pausing all jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java new file mode 100644 index 00000000..df6dac44 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class PauseQuartzJobsProcess extends AbstractLoadStep implements MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + String tableName = "quartzJobDetails"; + + return StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(getClass().getSimpleName()) + .withLabel("Pause Quartz Jobs") + .withPreviewMessage("This is a preview of the jobs that will be paused.") + .withTableName(tableName) + .withSourceTable(tableName) + .withDestinationTable(tableName) + .withExtractStepClass(ExtractViaQueryStep.class) + .withTransformStepClass(NoopTransformStep.class) + .withLoadStepClass(getClass()) + .withIcon(new QIcon("pause_circle_outline")) + .withReviewStepRecordFields(List.of( + new QFieldMetaData("id", QFieldType.LONG), + new QFieldMetaData("jobName", QFieldType.STRING), + new QFieldMetaData("jobGroup", QFieldType.STRING))) + .getProcessMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler instance = QuartzScheduler.getInstance(); + for(QRecord record : runBackendStepInput.getRecords()) + { + instance.pauseJob(record.getValueString("jobName"), record.getValueString("jobGroup")); + } + } + catch(Exception e) + { + throw (new QException("Error pausing jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java new file mode 100644 index 00000000..7d81e31f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** Manage process to resume all quartz jobs + *******************************************************************************/ +public class ResumeAllQuartzJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Resume All Quartz Jobs") + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to resume all quartz jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All quartz jobs have been resumed"))))) + .withIcon(new QIcon("play_circle_outline")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler.getInstance().resumeAll(); + } + catch(Exception e) + { + throw (new QException("Error resuming all jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java new file mode 100644 index 00000000..14c4cc27 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ResumeQuartzJobsProcess extends AbstractLoadStep implements MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + String tableName = "quartzJobDetails"; + + return StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(getClass().getSimpleName()) + .withLabel("Resume Quartz Jobs") + .withPreviewMessage("This is a preview of the jobs that will be resumed.") + .withTableName(tableName) + .withSourceTable(tableName) + .withDestinationTable(tableName) + .withExtractStepClass(ExtractViaQueryStep.class) + .withTransformStepClass(NoopTransformStep.class) + .withLoadStepClass(getClass()) + .withIcon(new QIcon("play_circle_outline")) + .withReviewStepRecordFields(List.of( + new QFieldMetaData("id", QFieldType.LONG), + new QFieldMetaData("jobName", QFieldType.STRING), + new QFieldMetaData("jobGroup", QFieldType.STRING))) + .getProcessMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler instance = QuartzScheduler.getInstance(); + for(QRecord record : runBackendStepInput.getRecords()) + { + instance.resumeJob(record.getValueString("jobName"), record.getValueString("jobGroup")); + } + } + catch(Exception e) + { + throw (new QException("Error resuming jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java new file mode 100644 index 00000000..f16cbb83 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.tables; + + +import java.time.Instant; +import java.time.ZoneId; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.apache.commons.lang3.SerializationUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzJobDataPostQueryCustomizer extends AbstractPostQueryCustomizer +{ + private static final QLogger LOG = QLogger.getLogger(QuartzJobDataPostQueryCustomizer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) + { + for(QRecord record : records) + { + if(record.getValue("jobData") != null) + { + try + { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // this field has a blob of essentially a serialized map - so, deserialize that, then convert to JSON // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + byte[] value = record.getValueByteArray("jobData"); + if(value.length > 0) + { + Object deserialize = SerializationUtils.deserialize(value); + String json = JsonUtils.toJson(deserialize); + record.setValue("jobData", json); + } + } + catch(Exception e) + { + LOG.info("Error deserializing quartz job data", e); + } + } + + formatEpochTime(record, "nextFireTime"); + formatEpochTime(record, "prevFireTime"); + formatEpochTime(record, "startTime"); + } + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void formatEpochTime(QRecord record, String fieldName) + { + Long value = record.getValueLong(fieldName); + + try + { + if(value != null && value > 0) + { + Instant instant = Instant.ofEpochMilli(value); + record.setDisplayValue(fieldName, String.format("%,d", value) + " (" + QValueFormatter.formatDateTimeWithZone(instant.atZone(ZoneId.of(QContext.getQInstance().getDefaultTimeZoneId()))) + ")"); + } + } + catch(Exception e) + { + LOG.info("Error formatting an epoc time value", e, logPair("fieldName", fieldName), logPair("value", value)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java new file mode 100644 index 00000000..8f4d04c1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java @@ -0,0 +1,98 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable; + + +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulableType +{ + private String name; + private QCodeReference runner; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public SchedulableType withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for runner + *******************************************************************************/ + public QCodeReference getRunner() + { + return (this.runner); + } + + + + /******************************************************************************* + ** Setter for runner + *******************************************************************************/ + public void setRunner(QCodeReference runner) + { + this.runner = runner; + } + + + + /******************************************************************************* + ** Fluent setter for runner + *******************************************************************************/ + public SchedulableType withRunner(QCodeReference runner) + { + this.runner = runner; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java new file mode 100644 index 00000000..2ad2a14d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +import java.util.Objects; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Basic implementation of interface for identifying schedulable things + *******************************************************************************/ +public class BasicSchedulableIdentity implements SchedulableIdentity +{ + private String identity; + private String description; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BasicSchedulableIdentity(String identity, String description) + { + if(!StringUtils.hasContent(identity)) + { + throw (new IllegalArgumentException("Identity may not be null or empty.")); + } + + this.identity = identity; + this.description = description; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + + if(o == null || getClass() != o.getClass()) + { + return false; + } + + BasicSchedulableIdentity that = (BasicSchedulableIdentity) o; + return Objects.equals(identity, that.identity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int hashCode() + { + return Objects.hash(identity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getIdentity() + { + return identity; + } + + + + /******************************************************************************* + ** Getter for description + ** + *******************************************************************************/ + @Override + public String getDescription() + { + return description; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return getIdentity(); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java new file mode 100644 index 00000000..f4cb1334 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +/******************************************************************************* + ** Unique identifier for a thing that can be scheduled + *******************************************************************************/ +public interface SchedulableIdentity +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + boolean equals(Object that); + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + int hashCode(); + + + /******************************************************************************* + ** + *******************************************************************************/ + String getIdentity(); + + + /******************************************************************************* + ** should NOT be part of equals & has code + *******************************************************************************/ + String getDescription(); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java new file mode 100644 index 00000000..0ad89408 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; + + +/******************************************************************************* + ** Factory to produce SchedulableIdentity objects + *******************************************************************************/ +public class SchedulableIdentityFactory +{ + + /******************************************************************************* + ** Factory to create one of these for a scheduled job record + *******************************************************************************/ + public static BasicSchedulableIdentity of(ScheduledJob scheduledJob) + { + String description = ""; + SchedulableType schedulableType = QContext.getQInstance().getSchedulableType(scheduledJob.getType()); + if(schedulableType != null) + { + try + { + SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + description = runner.getDescription(new HashMap<>(scheduledJob.getJobParametersMap())); + } + catch(Exception e) + { + description = "type: " + schedulableType.getName(); + } + } + + return new BasicSchedulableIdentity("scheduledJob:" + scheduledJob.getId(), description); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static BasicSchedulableIdentity of(QProcessMetaData process) + { + return new BasicSchedulableIdentity("process:" + process.getName(), "Process: " + process.getName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static SchedulableIdentity of(QQueueMetaData queue) + { + return new BasicSchedulableIdentity("queue:" + queue.getName(), "Queue: " + queue.getName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static SchedulableIdentity of(PollingAutomationPerTableRunner.TableActionsInterface tableActions) + { + return new BasicSchedulableIdentity("tableAutomations:" + tableActions.tableName() + "." + tableActions.status(), "TableAutomations: " + tableActions.tableName() + "." + tableActions.status()); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java new file mode 100644 index 00000000..c94d25ba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable process runner - e.g., how a QProcess is run by a scheduler. + *******************************************************************************/ +public class SchedulableProcessRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableProcessRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + String processName = ValueUtils.getValueAsString(params.get("processName")); + + /////////////////////////////////////// + // get the process from the instance // + /////////////////////////////////////// + QInstance qInstance = QContext.getQInstance(); + QProcessMetaData process = qInstance.getProcess(processName); + if(process == null) + { + LOG.warn("Could not find scheduled process in QInstance", logPair("processName", processName)); + return; + } + + /////////////////////////////////////////////// + // if the job has variant data, get it ready // + /////////////////////////////////////////////// + Map backendVariantData = null; + if(params.containsKey("backendVariantData")) + { + backendVariantData = (Map) params.get("backendVariantData"); + } + + Map processInputValues = buildProcessInputValuesMap(params, process); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running scheduled process", logPair("processName", processName)); + SchedulerUtils.runProcess(qInstance, () -> QContext.getQSession(), qInstance.getProcess(processName), backendVariantData, processInputValues); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String processName = ValueUtils.getValueAsString(paramMap.get("processName")); + if(!StringUtils.hasContent(processName)) + { + throw (new QException("Missing scheduledJobParameter with key [processName] in " + schedulableIdentity)); + } + + QProcessMetaData process = QContext.getQInstance().getProcess(processName); + if(process == null) + { + throw (new QException("Unrecognized processName [" + processName + "] in " + schedulableIdentity)); + } + + if(process.getSchedule() != null) + { + throw (new QException("Process [" + processName + "] has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "Process: " + params.get("processName"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Map buildProcessInputValuesMap(Map params, QProcessMetaData process) + { + Map processInputValues = new HashMap<>(); + + ////////////////////////////////////////////////////////////////////////////////////// + // track which keys need processed - start by removing ones we know we handle above // + ////////////////////////////////////////////////////////////////////////////////////// + Set keys = new HashSet<>(params.keySet()); + keys.remove("processName"); + keys.remove("backendVariantData"); + + if(!keys.isEmpty()) + { + ////////////////////////////////////////////////////////////////////////// + // first make a pass going over the process's identified input fields - // + // getting values from the quartz job data map, and putting them into // + // the process input value map as the field's type (if we can) // + ////////////////////////////////////////////////////////////////////////// + for(QFieldMetaData inputField : process.getInputFields()) + { + String fieldName = inputField.getName(); + if(params.containsKey(fieldName)) + { + Object value = params.get(fieldName); + try + { + processInputValues.put(fieldName, ValueUtils.getValueAsFieldType(inputField.getType(), value)); + keys.remove(fieldName); + } + catch(Exception e) + { + LOG.warn("Error getting process input value from quartz job data map", e, logPair("fieldName", fieldName), logPair("value", value)); + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////// + // if any values are left in the map (based on keys set that we're removing from) // + // then try to put those in the input map (assuming they can be cast to Serializable) // + //////////////////////////////////////////////////////////////////////////////////////// + for(String key : keys) + { + Object value = params.get(key); + try + { + processInputValues.put(key, (Serializable) value); + } + catch(Exception e) + { + LOG.warn("Error getting process input value from quartz job data map", e, logPair("key", key), logPair("value", value)); + } + } + } + + return processInputValues; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java new file mode 100644 index 00000000..b718d609 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; + + +/******************************************************************************* + ** Interface for different types of schedulabe things that can be run + *******************************************************************************/ +public interface SchedulableRunner +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void run(Map params); + + + /******************************************************************************* + ** + *******************************************************************************/ + void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException; + + /******************************************************************************* + ** + *******************************************************************************/ + String getDescription(Map params); +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java new file mode 100644 index 00000000..f2cc4dab --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java @@ -0,0 +1,135 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable SQSQueue runner - e.g., how an SQSQueuePoller is run by a scheduler. + *******************************************************************************/ +public class SchedulableSQSQueueRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableSQSQueueRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + String queueName = ValueUtils.getValueAsString(params.get("queueName")); + if(!StringUtils.hasContent(queueName)) + { + LOG.warn("Missing queueName in params."); + return; + } + + QQueueMetaData queue = qInstance.getQueue(queueName); + if(queue == null) + { + LOG.warn("Unrecognized queueName [" + queueName + "]"); + return; + } + + QQueueProviderMetaData queueProvider = qInstance.getQueueProvider(queue.getProviderName()); + if(!(queueProvider instanceof SQSQueueProviderMetaData)) + { + LOG.warn("Queue [" + queueName + "] is of an unsupported queue provider type (not SQS)"); + return; + } + + SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); + sqsQueuePoller.setQueueMetaData(queue); + sqsQueuePoller.setQueueProviderMetaData((SQSQueueProviderMetaData) queueProvider); + sqsQueuePoller.setQInstance(qInstance); + sqsQueuePoller.setSessionSupplier(QuartzScheduler.getInstance().getSessionSupplier()); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running SQS Queue poller", logPair("queueName", queueName)); + sqsQueuePoller.run(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String queueName = ValueUtils.getValueAsString(paramMap.get("queueName")); + if(!StringUtils.hasContent(queueName)) + { + throw (new QException("Missing scheduledJobParameter with key [queueName] in " + schedulableIdentity)); + } + + QQueueMetaData queue = QContext.getQInstance().getQueue(queueName); + if(queue == null) + { + throw (new QException("Unrecognized queueName [" + queueName + "] in " + schedulableIdentity)); + } + + QQueueProviderMetaData queueProvider = QContext.getQInstance().getQueueProvider(queue.getProviderName()); + if(!(queueProvider instanceof SQSQueueProviderMetaData)) + { + throw (new QException("Queue [" + queueName + "] is of an unsupported queue provider type (not SQS) in " + schedulableIdentity)); + } + + if(queue.getSchedule() != null) + { + throw (new QException("Queue [" + queueName + "] has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "Queue: " + params.get("queueName"); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java new file mode 100644 index 00000000..44bd2f20 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java @@ -0,0 +1,164 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable TableAutomations runner - e.g., how a table automations are run + ** by a scheduler. + *******************************************************************************/ +public class SchedulableTableAutomationsRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableTableAutomationsRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + String tableName = ValueUtils.getValueAsString(params.get("tableName")); + if(!StringUtils.hasContent(tableName)) + { + LOG.warn("Missing tableName in params."); + return; + } + + QTableMetaData table = qInstance.getTable(tableName); + if(table == null) + { + LOG.warn("Unrecognized tableName [" + tableName + "]"); + return; + } + + AutomationStatus automationStatus = AutomationStatus.valueOf(ValueUtils.getValueAsString(params.get("automationStatus"))); + + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails == null) + { + LOG.warn("Could not find automationDetails for table for automations in QInstance", logPair("tableName", tableName)); + return; + } + + /////////////////////////////////// + // todo - sharded automations... // + /////////////////////////////////// + PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationDetails, automationStatus); + PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationDetails.getProviderName(), QuartzScheduler.getInstance().getSessionSupplier(), tableAction); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running Table Automations", logPair("tableName", tableName), logPair("automationStatus", automationStatus)); + runner.run(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String tableName = ValueUtils.getValueAsString(paramMap.get("tableName")); + if(!StringUtils.hasContent(tableName)) + { + throw (new QException("Missing scheduledJobParameter with key [tableName] in " + schedulableIdentity)); + } + + String automationStatus = ValueUtils.getValueAsString(paramMap.get("automationStatus")); + if(!StringUtils.hasContent(automationStatus)) + { + throw (new QException("Missing scheduledJobParameter with key [automationStatus] in " + schedulableIdentity)); + } + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table == null) + { + throw (new QException("Unrecognized tableName [" + tableName + "] in " + schedulableIdentity)); + } + + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails == null) + { + throw (new QException("Table [" + tableName + "] does not have automationDetails in " + schedulableIdentity)); + } + + if(automationDetails.getSchedule() != null) + { + throw (new QException("Table [" + tableName + "] automationDetails has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + + QAutomationProviderMetaData automationProvider = QContext.getQInstance().getAutomationProvider(automationDetails.getProviderName()); + + List tableActionList = PollingAutomationPerTableRunner.getTableActions(QContext.getQInstance(), automationProvider.getName()); + for(PollingAutomationPerTableRunner.TableActionsInterface tableActions : tableActionList) + { + if(tableActions.status().name().equals(automationStatus)) + { + return; + } + } + + ///////////////////////////////////////////////////////////////////////////////////// + // if we get out of the loop, it means we didn't find a matching status - so throw // + ///////////////////////////////////////////////////////////////////////////////////// + throw (new QException("Did not find table automation actions matching automationStatus [" + automationStatus + "] for table [" + tableName + "] in " + schedulableIdentity + + " (Found: " + tableActionList.stream().map(ta -> ta.status().name()).collect(Collectors.joining(",")) + ")")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "TableAutomations: " + params.get("tableName") + "." + params.get("automationStatus"); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java new file mode 100644 index 00000000..bbdf6466 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.simple; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SimpleJobRunner implements Runnable +{ + private static final QLogger LOG = QLogger.getLogger(SimpleJobRunner.class); + + private QInstance qInstance; + private SchedulableType schedulableType; + private Map params; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SimpleJobRunner(QInstance qInstance, SchedulableType type, Map params) + { + this.qInstance = qInstance; + this.schedulableType = type; + this.params = params; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run() + { + CapturedContext capturedContext = QContext.capture(); + try + { + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + QContext.init(qInstance, simpleScheduler.getSessionSupplier().get()); + + SchedulableRunner schedulableRunner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + schedulableRunner.run(params); + } + catch(Exception e) + { + LOG.warn("Error running SimpleScheduler job", e, logPair("params", params)); + } + finally + { + QContext.init(capturedContext); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java new file mode 100644 index 00000000..8574fecc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -0,0 +1,311 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.simple; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** QQQ Service (Singleton) that starts up repeating, scheduled jobs within QQQ. + ** + ** These include: + ** - Automation providers (which require polling) + ** - Queue pollers + ** - Scheduled processes. + ** + ** All of these jobs run using a "system session" - as defined by the sessionSupplier. + *******************************************************************************/ +public class SimpleScheduler implements QSchedulerInterface +{ + private static final QLogger LOG = QLogger.getLogger(SimpleScheduler.class); + + private static SimpleScheduler simpleScheduler = null; + private final QInstance qInstance; + private String schedulerName; + + protected Supplier sessionSupplier; + + ///////////////////////////////////////////////////////////////////////////////////// + // for jobs that don't define a delay index, auto-stagger them, using this counter // + ///////////////////////////////////////////////////////////////////////////////////// + private int delayIndex = 0; + + private Map executors = new LinkedHashMap<>(); + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private SimpleScheduler(QInstance qInstance) + { + this.qInstance = qInstance; + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static SimpleScheduler getInstance(QInstance qInstance) + { + if(simpleScheduler == null) + { + simpleScheduler = new SimpleScheduler(qInstance); + } + return (simpleScheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void start() + { + for(StandardScheduledExecutor executor : executors.values()) + { + executor.start(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stopAsync() + { + for(StandardScheduledExecutor scheduledExecutor : executors.values()) + { + scheduledExecutor.stopAsync(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stop() + { + for(StandardScheduledExecutor scheduledExecutor : executors.values()) + { + scheduledExecutor.stop(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart) + { + if(!allowedToStart) + { + return; + } + + SimpleJobRunner simpleJobRunner = new SimpleJobRunner(qInstance, schedulableType, new HashMap<>(parameters)); + StandardScheduledExecutor executor = new StandardScheduledExecutor(simpleJobRunner); + executor.setName(schedulableIdentity.getIdentity()); + setScheduleInExecutor(schedule, executor); + executors.put(schedulableIdentity, executor); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setScheduleInExecutor(QScheduleMetaData schedule, StandardScheduledExecutor executor) + { + if(schedule.getRepeatMillis() != null) + { + executor.setDelayMillis(schedule.getRepeatMillis()); + } + else + { + executor.setDelayMillis(1000 * schedule.getRepeatSeconds()); + } + + if(schedule.getInitialDelayMillis() != null) + { + executor.setInitialDelayMillis(schedule.getInitialDelayMillis()); + } + else if(schedule.getInitialDelaySeconds() != null) + { + executor.setInitialDelayMillis(1000 * schedule.getInitialDelaySeconds()); + } + else + { + executor.setInitialDelayMillis(1000 * ++delayIndex); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + StandardScheduledExecutor executor = executors.get(schedulableIdentity); + if(executor != null) + { + LOG.info("Stopping job in simple scheduler", logPair("identity", schedulableIdentity)); + executors.remove(schedulableIdentity); + executor.stop(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + return (executors.containsKey(schedulableIdentity)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleAll() throws QException + { + for(Map.Entry entry : new HashSet<>(executors.entrySet())) + { + StandardScheduledExecutor executor = executors.remove(entry.getKey()); + if(executor != null) + { + executor.stopAsync(); + } + } + } + + + + /******************************************************************************* + ** Setter for sessionSupplier + ** + *******************************************************************************/ + public void setSessionSupplier(Supplier sessionSupplier) + { + this.sessionSupplier = sessionSupplier; + } + + + + /******************************************************************************* + ** Getter for sessionSupplier + ** + *******************************************************************************/ + public Supplier getSessionSupplier() + { + return sessionSupplier; + } + + + + /******************************************************************************* + ** Getter for managedExecutors + ** + *******************************************************************************/ + public List getExecutors() + { + return new ArrayList<>(executors.values()); + } + + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public SimpleScheduler withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unInit() + { + ////////////////////////////////////////////////// + // resetting the singleton should be sufficient // + ////////////////////////////////////////////////// + simpleScheduler = null; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java similarity index 95% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java index 3910e65f..36c858f5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; import java.util.concurrent.Executors; @@ -32,8 +32,10 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; /******************************************************************************* - ** Standard class ran by ScheduleManager. Takes a Runnable in its constructor - - ** that's the code that actually executes. + ** Standard class ran by SimpleScheduler. Takes a Runnable in its constructor - + ** that's the code that actually executes. Internally, this class will launch + ** a newSingleThreadScheduledExecutor / ScheduledExecutorService to run the + ** runnable on a repeating delay. ** *******************************************************************************/ public class StandardScheduledExecutor diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java index f6fe0e1a..e7a6db09 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/AbstractStateKey.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.state; import java.io.Serializable; +import java.time.Instant; /******************************************************************************* @@ -57,4 +58,10 @@ public abstract class AbstractStateKey implements Serializable @Override public abstract String toString(); + /******************************************************************************* + ** Require all state keys to implement the getStartTime method + * + *******************************************************************************/ + public abstract Instant getStartTime(); + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java index 73f54d12..52d756ec 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java @@ -23,9 +23,16 @@ package com.kingsrook.qqq.backend.core.state; import java.io.Serializable; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -33,10 +40,16 @@ import java.util.Optional; *******************************************************************************/ public class InMemoryStateProvider implements StateProviderInterface { + private static final QLogger LOG = QLogger.getLogger(InMemoryStateProvider.class); + private static InMemoryStateProvider instance; private final Map map; + private static int jobPeriodSeconds = 60 * 30; // 30 minutes + private static int cleanHours = 5; + private static int jobInitialDelay = 60 * 60 * cleanHours; + /******************************************************************************* @@ -45,6 +58,41 @@ public class InMemoryStateProvider implements StateProviderInterface private InMemoryStateProvider() { this.map = new HashMap<>(); + + /////////////////////////////////////////////////////////// + // Start a single thread executor to handle the cleaning // + /////////////////////////////////////////////////////////// + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + executorService.scheduleAtFixedRate(new InMemoryStateProvider.InMemoryStateProviderCleanJob(), jobInitialDelay, jobPeriodSeconds, TimeUnit.SECONDS); + } + + + + /******************************************************************************* + ** Runnable that gets scheduled to periodically clean the InMemoryStateProvider + *******************************************************************************/ + private static class InMemoryStateProviderCleanJob implements Runnable + { + private static final QLogger LOG = QLogger.getLogger(InMemoryStateProvider.InMemoryStateProviderCleanJob.class); + + + + /******************************************************************************* + ** run + *******************************************************************************/ + @Override + public void run() + { + try + { + Instant cleanTime = Instant.now().minus(cleanHours, ChronoUnit.HOURS); + getInstance().clean(cleanTime); + } + catch(Exception e) + { + LOG.warn("Error cleaning InMemoryStateProvider entries.", e); + } + } } @@ -101,4 +149,36 @@ public class InMemoryStateProvider implements StateProviderInterface map.remove(key); } + + + /******************************************************************************* + ** Get the current status + * + *******************************************************************************/ + @Override + public String status() + { + return ("InMemoryStateProvider map size: " + map.size()); + } + + + + /******************************************************************************* + ** Clean entries that started before the given Instant + * + *******************************************************************************/ + @Override + public void clean(Instant cleanBeforeInstant) + { + long jobStartTime = System.currentTimeMillis(); + Integer beforeSize = map.size(); + LOG.info("Starting clean for InMemoryStateProvider.", logPair("beforeSize", beforeSize)); + + map.entrySet().removeIf(e -> e.getKey().getStartTime().isBefore(cleanBeforeInstant)); + + Integer afterSize = map.size(); + long endTime = System.currentTimeMillis(); + LOG.info("Completed clean for InMemoryStateProvider.", logPair("beforeSize", beforeSize), logPair("afterSize", afterSize), logPair("amountCleaned", (beforeSize - afterSize)), logPair("runTimeMillis", (endTime - jobStartTime))); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/SimpleStateKey.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/SimpleStateKey.java index 61f19fdb..7a3fefe1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/SimpleStateKey.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/SimpleStateKey.java @@ -22,6 +22,9 @@ package com.kingsrook.qqq.backend.core.state; +import java.time.Instant; + + /******************************************************************************* ** *******************************************************************************/ @@ -93,4 +96,17 @@ public class SimpleStateKey extends AbstractStateKey { return key.hashCode(); } + + + + /******************************************************************************* + ** Getter for startTime + *******************************************************************************/ + public Instant getStartTime() + { + ////////////////////////////////////////// + // For now these will never get cleaned // + ////////////////////////////////////////// + return (Instant.now()); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java index 1c5415bf..43794edd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.state; import java.io.Serializable; +import java.time.Instant; import java.util.Optional; @@ -58,4 +59,13 @@ public interface StateProviderInterface *******************************************************************************/ void remove(AbstractStateKey key); + /******************************************************************************* + ** Get the current status + *******************************************************************************/ + String status(); + + /******************************************************************************* + ** Clean entries that started before the given Instant + *******************************************************************************/ + void clean(Instant startTime); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java index b9f6ed5e..7f091f1a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java @@ -26,6 +26,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Serializable; +import java.nio.file.NoSuchFileException; +import java.time.Instant; import java.util.Optional; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.utils.JsonUtils; @@ -98,14 +100,14 @@ public class TempFileStateProvider implements StateProviderInterface String json = FileUtils.readFileToString(getFile(key)); return (Optional.of(JsonUtils.toObject(json, type))); } - catch(FileNotFoundException fnfe) + catch(FileNotFoundException | NoSuchFileException fnfe) { return (Optional.empty()); } catch(IOException e) { LOG.error("Error getting state from file", e); - throw (new RuntimeException("Error retreiving state", e)); + throw (new RuntimeException("Error retrieving state", e)); } } @@ -126,6 +128,31 @@ public class TempFileStateProvider implements StateProviderInterface + /******************************************************************************* + ** Get the current status + * + *******************************************************************************/ + @Override + public String status() + { + return ("TempFileStateProvider status not supported"); + } + + + + /******************************************************************************* + ** Clean entries that started before the given Instant + *******************************************************************************/ + @Override + public void clean(Instant startTime) + { + //////////////////////////////// + // Not supported at this time // + //////////////////////////////// + } + + + /******************************************************************************* ** Get the file referenced by a key *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java index 38659ccc..24486be6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.state; import java.io.Serializable; +import java.time.Instant; import java.util.Objects; import java.util.UUID; @@ -34,6 +35,7 @@ public class UUIDAndTypeStateKey extends AbstractStateKey implements Serializabl { private final UUID uuid; private final StateType stateType; + private final Instant startTime; @@ -43,7 +45,7 @@ public class UUIDAndTypeStateKey extends AbstractStateKey implements Serializabl *******************************************************************************/ public UUIDAndTypeStateKey(StateType stateType) { - this(UUID.randomUUID(), stateType); + this(UUID.randomUUID(), stateType, Instant.now()); } @@ -53,9 +55,21 @@ public class UUIDAndTypeStateKey extends AbstractStateKey implements Serializabl ** *******************************************************************************/ public UUIDAndTypeStateKey(UUID uuid, StateType stateType) + { + this(uuid, stateType, Instant.now()); + } + + + + /******************************************************************************* + ** Constructor where user can supply the UUID. + ** + *******************************************************************************/ + public UUIDAndTypeStateKey(UUID uuid, StateType stateType, Instant startTime) { this.uuid = uuid; this.stateType = stateType; + this.startTime = startTime; } @@ -133,4 +147,15 @@ public class UUIDAndTypeStateKey extends AbstractStateKey implements Serializabl { return "{uuid=" + uuid + ", stateType=" + stateType + '}'; } + + + + /******************************************************************************* + ** Getter for startTime + *******************************************************************************/ + public Instant getStartTime() + { + return (this.startTime); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java index 95b8d091..1e53c346 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java @@ -518,6 +518,24 @@ public class CollectionUtils + /******************************************************************************* + ** Convert a collection of QRecords to a map, from one field's values out of + ** those records, to the records themselves. + *******************************************************************************/ + public static Map recordsToMap(Collection records, String keyFieldName, Class type) + { + Map rs = new HashMap<>(); + + for(QRecord record : nonNullCollection(records)) + { + rs.put(ValueUtils.getValueAsType(type, record.getValue(keyFieldName)), record); + } + + return (rs); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java index a2a36a37..7c000e3e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java @@ -244,6 +244,7 @@ public class JsonUtils .registerModule(new JavaTimeModule()) .setSerializationInclusion(JsonInclude.Include.NON_NULL) .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) + .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); /* todo - some future version we may need to do inclusion/exclusion lists like this: @@ -362,6 +363,7 @@ public class JsonUtils switch(metaData.getType()) { case INTEGER -> record.setValue(fieldName, jsonObjectToUse.optInt(backendName)); + case LONG -> record.setValue(fieldName, jsonObjectToUse.optLong(backendName)); case DECIMAL -> record.setValue(fieldName, jsonObjectToUse.optBigDecimal(backendName, null)); case BOOLEAN -> record.setValue(fieldName, jsonObjectToUse.optBoolean(backendName)); case DATE_TIME -> diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/LocalMacDevUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/LocalMacDevUtils.java new file mode 100644 index 00000000..d2725045 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/LocalMacDevUtils.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils; + + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import com.kingsrook.qqq.backend.core.logging.QLogger; + + +/******************************************************************************* + ** Useful things to do on a mac, when doing development - that we can expect + ** may not exist in a prod or even CI environment. So, they'll only happen if + ** flags are set to do them, and if we're on a mac (e.g., paths exist) + *******************************************************************************/ +public class LocalMacDevUtils +{ + private static final QLogger LOG = QLogger.getLogger(LocalMacDevUtils.class); + + public static boolean mayOpenFiles = false; + + private static final String OPEN_PROGRAM_PATH = "/usr/bin/open"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void openFile(String path) throws IOException + { + if(mayOpenFiles && Files.exists(Path.of(OPEN_PROGRAM_PATH))) + { + Runtime.getRuntime().exec(new String[] { OPEN_PROGRAM_PATH, path }); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void openFile(String path, String appPath) throws IOException + { + if(mayOpenFiles && Files.exists(Path.of(OPEN_PROGRAM_PATH))) + { + if(Files.exists(Path.of(appPath))) + { + Runtime.getRuntime().exec(new String[] { OPEN_PROGRAM_PATH, "-a", appPath, path }); + } + else + { + LOG.warn("App at path [" + appPath + " was not found - file [" + path + "] will not be opened."); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ObjectUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ObjectUtils.java index 80901c44..0fa1566e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ObjectUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ObjectUtils.java @@ -139,4 +139,28 @@ public class ObjectUtils return (b); } + + + /******************************************************************************* + ** Utility to test a chained unsafe expression CAN get to the end and return true. + ** + ** e.g., instead of: + ** if(a && a.b && a.b.c && a.b.c.d) + ** we can do: + ** if(ifCan(() -> a.b.c.d)) + ** + ** Note - if the supplier returns null, that counts as false! + *******************************************************************************/ + public static boolean ifCan(UnsafeSupplier supplier) + { + try + { + return supplier.get(); + } + catch(Throwable t) + { + return (false); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduper.java index b6d6e0d0..3370b68e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduper.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IS_NOT_BLANK; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_EQUALS; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_IN; @@ -311,6 +312,28 @@ public class QQueryFilterDeduper log.add("Merge two not-equals as not-in"); continue; } + else if(IN.equals(other.getOperator()) && IS_NOT_BLANK.equals(criteria.getOperator())) + { + ////////////////////////////////////////////////////////////////////////// + // for an IN and IS_NOT_BLANK, remove the IS_NOT_BLANK - it's redundant // + ////////////////////////////////////////////////////////////////////////// + iterator.remove(); + didAnyGood = true; + log.add("Removing redundant is-not-blank"); + continue; + } + else if(IS_NOT_BLANK.equals(other.getOperator()) && IN.equals(criteria.getOperator())) + { + ////////////////////////////////////////////////////////////////////////// + // for an IN and IS_NOT_BLANK, remove the IS_NOT_BLANK - it's redundant // + ////////////////////////////////////////////////////////////////////////// + other.setOperator(IN); + other.setValues(new ArrayList<>(criteria.getValues())); + iterator.remove(); + didAnyGood = true; + log.add("Removing redundant is-not-blank"); + continue; + } else { log.add("Fail because unhandled operator pair"); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/SleepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/SleepUtils.java index 4487821a..5d374257 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/SleepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/SleepUtils.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.utils; +import java.time.Duration; import java.util.concurrent.TimeUnit; @@ -55,4 +56,14 @@ public class SleepUtils } } + + + /******************************************************************************* + ** overload for sleep that takes duration object + *******************************************************************************/ + public static void sleep(Duration sleepDuration) + { + sleep(sleepDuration.toMillis(), TimeUnit.MILLISECONDS); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValidationUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValidationUtils.java new file mode 100644 index 00000000..4674f66f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValidationUtils.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils; + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import org.apache.commons.validator.EmailValidator; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ValidationUtils +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static List parseAndValidateEmailAddresses(String emailAddresses) throws QUserFacingException + { + //////////////////////////////////////////////////////////////// + // split email address string on spaces, comma, and semicolon // + //////////////////////////////////////////////////////////////// + List toEmailAddressList = Arrays.asList(emailAddresses.split("[\\s,;]+")); + + ////////////////////////////////////////////////////// + // check each address keeping track of any bad ones // + ////////////////////////////////////////////////////// + List invalidEmails = new ArrayList<>(); + EmailValidator validator = EmailValidator.getInstance(); + for(String emailAddress : toEmailAddressList) + { + if(!validator.isValid(emailAddress)) + { + invalidEmails.add(emailAddress); + } + } + + /////////////////////////////////////// + // if bad one found, throw exception // + /////////////////////////////////////// + if(!invalidEmails.isEmpty()) + { + throw (new QUserFacingException("The following email addresses were invalid: " + StringUtils.join(",", invalidEmails))); + } + + return (toEmailAddressList); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 8bc4553a..a2bd7bca 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -114,6 +114,113 @@ public class ValueUtils + /******************************************************************************* + ** Type-safely make an Long from any Object. + ** null and empty-string inputs return null. + ** We try to strip away commas and decimals (as long as they are exactly equal to the int value) + ** We may throw if the input can't be converted to an integer. + *******************************************************************************/ + public static Long getValueAsLong(Object value) throws QValueException + { + try + { + if(value == null) + { + return (null); + } + else if(value instanceof Integer i) + { + return Long.valueOf((i)); + } + else if(value instanceof Long l) + { + return (l); + } + else if(value instanceof BigInteger b) + { + return (b.longValue()); + } + else if(value instanceof Float f) + { + if(f.longValue() != f) + { + throw (new QValueException(f + " does not have an exact integer representation.")); + } + return (f.longValue()); + } + else if(value instanceof Double d) + { + if(d.longValue() != d) + { + throw (new QValueException(d + " does not have an exact integer representation.")); + } + return (d.longValue()); + } + else if(value instanceof BigDecimal bd) + { + return bd.longValueExact(); + } + else if(value instanceof PossibleValueEnum pve) + { + return getValueAsLong(pve.getPossibleValueId()); + } + else if(value instanceof String s) + { + if(!StringUtils.hasContent(s)) + { + return (null); + } + + try + { + return (Long.parseLong(s)); + } + catch(NumberFormatException nfe) + { + if(s.contains(",")) + { + String sWithoutCommas = s.replaceAll(",", ""); + try + { + return (getValueAsLong(sWithoutCommas)); + } + catch(Exception ignore) + { + throw (nfe); + } + } + if(s.matches(".*\\.\\d+$")) + { + String sWithoutDecimal = s.replaceAll("\\.\\d+$", ""); + try + { + return (getValueAsLong(sWithoutDecimal)); + } + catch(Exception ignore) + { + throw (nfe); + } + } + throw (nfe); + } + } + else + { + throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to Long.")); + } + } + catch(QValueException qve) + { + throw (qve); + } + catch(Exception e) + { + throw (new QValueException("Value [" + value + "] could not be converted to a Long.", e)); + } + } + + + /******************************************************************************* ** Type-safely make an Integer from any Object. ** null and empty-string inputs return null. @@ -686,13 +793,13 @@ public class ValueUtils /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") public static Serializable getValueAsFieldType(QFieldType type, Object value) { return switch(type) { case STRING, TEXT, HTML, PASSWORD -> getValueAsString(value); case INTEGER -> getValueAsInteger(value); + case LONG -> getValueAsLong(value); case DECIMAL -> getValueAsBigDecimal(value); case BOOLEAN -> getValueAsBoolean(value); case DATE -> getValueAsLocalDate(value); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java index 1c7a8f53..c52bf027 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.utils; import java.util.Map; +import java.util.function.Consumer; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -56,6 +57,16 @@ public class YamlUtils ** *******************************************************************************/ public static String toYaml(Object object) + { + return toYaml(object, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String toYaml(Object object, Consumer objectMapperCustomizer) { try { @@ -66,7 +77,10 @@ public class YamlUtils objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - // todo? objectMapper.setFilterProvider(new OmitDefaultValuesFilterProvider()); + if(objectMapperCustomizer != null) + { + objectMapperCustomizer.accept(objectMapper); + } objectMapper.findAndRegisterModules(); return (objectMapper.writeValueAsString(object)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java index 074c2469..ee6b0a59 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java @@ -29,8 +29,12 @@ import java.math.BigDecimal; /******************************************************************************* ** Classes that support doing data aggregations (e.g., count, sum, min, max, average). ** Sub-classes should supply the type parameter. + ** + ** The AVG_T parameter describes the type used for the average getAverage method + ** which, e.g, for date types, might be a date, vs. numbers, they'd probably be + ** BigDecimal. *******************************************************************************/ -public interface AggregatesInterface +public interface AggregatesInterface { /******************************************************************************* ** @@ -60,5 +64,51 @@ public interface AggregatesInterface /******************************************************************************* ** *******************************************************************************/ - BigDecimal getAverage(); + AVG_T getAverage(); + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getProduct() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getVariance() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getVarP() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getStandardDeviation() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getStdDevP() + { + return (null); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java index da7f1703..76a6f0d8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java @@ -28,13 +28,16 @@ import java.math.BigDecimal; /******************************************************************************* ** BigDecimal version of data aggregator *******************************************************************************/ -public class BigDecimalAggregates implements AggregatesInterface +public class BigDecimalAggregates implements AggregatesInterface { private int count = 0; // private Integer countDistinct; private BigDecimal sum; private BigDecimal min; private BigDecimal max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); @@ -59,6 +62,15 @@ public class BigDecimalAggregates implements AggregatesInterface sum = sum.add(input); } + if(product == null) + { + product = input; + } + else + { + product = product.multiply(input); + } + if(min == null || input.compareTo(min) < 0) { min = input; @@ -68,6 +80,52 @@ public class BigDecimalAggregates implements AggregatesInterface { max = input; } + + varianceCalculator.updateVariance(input); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); } @@ -116,6 +174,18 @@ public class BigDecimalAggregates implements AggregatesInterface + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java new file mode 100644 index 00000000..adb1a591 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java @@ -0,0 +1,136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigInteger; +import java.time.Instant; + + +/******************************************************************************* + ** Instant version of data aggregator + *******************************************************************************/ +public class InstantAggregates implements AggregatesInterface +{ + private int count = 0; + // private Integer countDistinct; + + private BigInteger sumMillis = BigInteger.ZERO; + + private Instant min; + private Instant max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(Instant input) + { + if(input == null) + { + return; + } + + count++; + + sumMillis = sumMillis.add(new BigInteger(String.valueOf(input.toEpochMilli()))); + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getSum() + { + ////////////////////////////////////////// + // sum of date-times doesn't make sense // + ////////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getAverage() + { + if(this.count > 0) + { + BigInteger averageMillis = this.sumMillis.divide(new BigInteger(String.valueOf(count))); + if(averageMillis.compareTo(new BigInteger(String.valueOf(Long.MAX_VALUE))) < 0) + { + return (Instant.ofEpochMilli(averageMillis.longValue())); + } + } + + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java index 292e8a01..15efecea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java @@ -28,13 +28,16 @@ import java.math.BigDecimal; /******************************************************************************* ** Integer version of data aggregator *******************************************************************************/ -public class IntegerAggregates implements AggregatesInterface +public class IntegerAggregates implements AggregatesInterface { - private int count = 0; + private int count = 0; // private Integer countDistinct; - private Integer sum; - private Integer min; - private Integer max; + private Integer sum; + private Integer min; + private Integer max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); @@ -48,6 +51,8 @@ public class IntegerAggregates implements AggregatesInterface return; } + BigDecimal inputBD = new BigDecimal(input); + count++; if(sum == null) @@ -59,6 +64,15 @@ public class IntegerAggregates implements AggregatesInterface sum = sum + input; } + if(product == null) + { + product = inputBD; + } + else + { + product = product.multiply(inputBD); + } + if(min == null || input < min) { min = input; @@ -68,6 +82,52 @@ public class IntegerAggregates implements AggregatesInterface { max = input; } + + varianceCalculator.updateVariance(inputBD); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); } @@ -116,6 +176,18 @@ public class IntegerAggregates implements AggregatesInterface + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java new file mode 100644 index 00000000..3c64e200 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java @@ -0,0 +1,136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigInteger; +import java.time.LocalDate; + + +/******************************************************************************* + ** LocalDate version of data aggregator + *******************************************************************************/ +public class LocalDateAggregates implements AggregatesInterface +{ + private int count = 0; + // private Integer countDistinct; + + private BigInteger sumMillis = BigInteger.ZERO; + + private LocalDate min; + private LocalDate max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(LocalDate input) + { + if(input == null) + { + return; + } + + count++; + + sumMillis = sumMillis.add(new BigInteger(String.valueOf(input.toEpochDay()))); + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getSum() + { + ////////////////////////////////////////// + // sum of date-times doesn't make sense // + ////////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getAverage() + { + if(this.count > 0) + { + BigInteger averageEpochDay = this.sumMillis.divide(new BigInteger(String.valueOf(count))); + if(averageEpochDay.compareTo(new BigInteger(String.valueOf(Long.MAX_VALUE))) < 0) + { + return (LocalDate.ofEpochDay(averageEpochDay.longValue())); + } + } + + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java new file mode 100644 index 00000000..e131cda1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java @@ -0,0 +1,207 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigDecimal; + + +/******************************************************************************* + ** Long version of data aggregator + *******************************************************************************/ +public class LongAggregates implements AggregatesInterface +{ + private int count = 0; + // private Long countDistinct; + private Long sum; + private Long min; + private Long max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(Long input) + { + if(input == null) + { + return; + } + + BigDecimal inputBD = new BigDecimal(input); + + count++; + + if(sum == null) + { + sum = input; + } + else + { + sum = sum + input; + } + + if(product == null) + { + product = inputBD; + } + else + { + product = product.multiply(inputBD); + } + + if(min == null || input < min) + { + min = input; + } + + if(max == null || input > max) + { + max = input; + } + + varianceCalculator.updateVariance(inputBD); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getSum() + { + return (sum); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getMax() + { + return (max); + } + + + + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getAverage() + { + if(this.count > 0) + { + return (BigDecimal.valueOf(this.sum.doubleValue() / (double) this.count)); + } + else + { + return (null); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java new file mode 100644 index 00000000..33e306f1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +/******************************************************************************* + ** String version of data aggregator + *******************************************************************************/ +public class StringAggregates implements AggregatesInterface +{ + private int count = 0; + + private String min; + private String max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(String input) + { + if(input == null) + { + return; + } + + count++; + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getSum() + { + ////////////////////////////////////// + // sum of string doesn't make sense // + ////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getAverage() + { + /////////////////////////////////////// + // average string doesn't make sense // + /////////////////////////////////////// + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java new file mode 100644 index 00000000..eefe04f6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigDecimal; +import java.math.RoundingMode; + + +/******************************************************************************* + ** see https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + ** + *******************************************************************************/ +public class VarianceCalculator +{ + private int n; + private BigDecimal runningMean = BigDecimal.ZERO; + private BigDecimal m2 = BigDecimal.ZERO; + + public static int scaleForVarianceCalculations = 4; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void updateVariance(BigDecimal newInput) + { + n++; + BigDecimal delta = newInput.subtract(runningMean); + runningMean = runningMean.add(delta.divide(new BigDecimal(n), scaleForVarianceCalculations, RoundingMode.HALF_UP)); + BigDecimal delta2 = newInput.subtract(runningMean); + m2 = m2.add(delta.multiply(delta2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getVariance() + { + if(n < 2) + { + return (null); + } + + return m2.divide(new BigDecimal(n - 1), scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getVarP() + { + if(n < 2) + { + return (null); + } + + return m2.divide(new BigDecimal(n), scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getStandardDeviation() + { + BigDecimal variance = getVariance(); + if(variance == null) + { + return (null); + } + + return BigDecimal.valueOf(Math.sqrt(variance.doubleValue())).setScale(scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getStdDevP() + { + BigDecimal varP = getVarP(); + if(varP == null) + { + return (null); + } + + return BigDecimal.valueOf(Math.sqrt(varP.doubleValue())).setScale(scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java new file mode 100644 index 00000000..e448d2e9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java @@ -0,0 +1,37 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.lambdas; + + +/******************************************************************************* + ** + *******************************************************************************/ +@FunctionalInterface +public interface UnsafeVoidVoidMethod +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void run() throws T; + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java index ae373024..4ed6cbce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java @@ -43,8 +43,9 @@ public class Memoization private final Map> map = Collections.synchronizedMap(new LinkedHashMap<>()); - private Duration timeout = Duration.ofSeconds(600); - private Integer maxSize = 1000; + private Duration timeout = Duration.ofSeconds(600); + private Integer maxSize = 1000; + private boolean mayStoreNullValues = true; @@ -58,6 +59,40 @@ public class Memoization + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Memoization(Integer maxSize) + { + this.maxSize = maxSize; + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Memoization(Duration timeout) + { + this.timeout = timeout; + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Memoization(Duration timeout, Integer maxSize) + { + this.timeout = timeout; + this.maxSize = maxSize; + } + + + /******************************************************************************* ** Get the memoized Value for a given input Key - computing it if it wasn't previously ** memoized (or expired). @@ -153,6 +188,14 @@ public class Memoization *******************************************************************************/ public void storeResult(K key, V value) { + ////////////////////////////////////////////////////////////////////////////////////////// + // if the value is null, and we're not supposed to store nulls, then return w/o storing // + ////////////////////////////////////////////////////////////////////////////////////////// + if(value == null && !mayStoreNullValues) + { + return; + } + map.put(key, new MemoizedResult<>(value)); ////////////////////////////////////// @@ -277,4 +320,35 @@ public class Memoization return (this); } + + + /******************************************************************************* + ** Getter for mayStoreNullValues + *******************************************************************************/ + public boolean getMayStoreNullValues() + { + return (this.mayStoreNullValues); + } + + + + /******************************************************************************* + ** Setter for mayStoreNullValues + *******************************************************************************/ + public void setMayStoreNullValues(boolean mayStoreNullValues) + { + this.mayStoreNullValues = mayStoreNullValues; + } + + + + /******************************************************************************* + ** Fluent setter for mayStoreNullValues + *******************************************************************************/ + public Memoization withMayStoreNullValues(boolean mayStoreNullValues) + { + this.mayStoreNullValues = mayStoreNullValues; + return (this); + } + } diff --git a/qqq-backend-core/src/main/resources/log4j2.xml b/qqq-backend-core/src/main/resources/log4j2.xml index 349d47d1..8883de80 100644 --- a/qqq-backend-core/src/main/resources/log4j2.xml +++ b/qqq-backend-core/src/main/resources/log4j2.xml @@ -26,6 +26,13 @@ + + + + + + + diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java index 27513ccb..be72d887 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java @@ -22,6 +22,8 @@ package com.kingsrook.qqq.backend.core; +import java.time.ZoneId; +import java.util.TimeZone; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -40,7 +42,12 @@ public class BaseTest { private static final QLogger LOG = QLogger.getLogger(BaseTest.class); + public static final String DEFAULT_USER_ID = "001"; + static + { + TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("UTC"))); + } /******************************************************************************* ** @@ -50,15 +57,34 @@ public class BaseTest { System.setProperty("qqq.logger.logSessionId.disabled", "true"); - QContext.init(TestUtils.defineInstance(), new QSession() - .withUser(new QUser() - .withIdReference("001") - .withFullName("Anonymous"))); + QContext.init(TestUtils.defineInstance(), newSession()); resetMemoryRecordStore(); } + /******************************************************************************* + ** + *******************************************************************************/ + protected QSession newSession() + { + return newSession(DEFAULT_USER_ID); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected QSession newSession(String userId) + { + return new QSession().withUser(new QUser() + .withIdReference(userId) + .withFullName("Anonymous")); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java index dc276b0d..c4e47381 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java @@ -575,8 +575,8 @@ class PollingAutomationPerTableRunnerTest extends BaseTest @Test void testLoadingRecordTypesToEnsureClassCoverage() { - new PollingAutomationPerTableRunner.TableActions(null, null).noopToFakeTestCoverage(); - new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null).noopToFakeTestCoverage(); + new PollingAutomationPerTableRunner.TableActions(null, null, null).noopToFakeTestCoverage(); + new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null, null).noopToFakeTestCoverage(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java index 57e0055d..e08b4ac1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java @@ -43,7 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; -import com.kingsrook.qqq.backend.core.scheduler.StandardScheduledExecutor; +import com.kingsrook.qqq.backend.core.scheduler.simple.StandardScheduledExecutor; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java new file mode 100644 index 00000000..83cf8298 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.utils.Timer; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for QCodeLoader + *******************************************************************************/ +class QCodeLoaderTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetAdHoc() + { + QCodeLoader qCodeLoader = QCodeLoader.getAdHoc(QCodeLoader.class, new QCodeReference(QCodeLoader.class)); + assertThat(qCodeLoader).isInstanceOf(QCodeLoader.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @Disabled("performance test, used during memoization change") + void testBulkPerformance() + { + Timer timer = new Timer("start"); + for(int i = 0; i < 5; i++) + { + useCodeLoader(1_000_000); + timer.mark("done with code loader"); + + useNew(1_000_000); + timer.mark("done with new"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void useNew(int count) + { + for(int i = 0; i < count; i++) + { + QCodeLoader qCodeLoader = new QCodeLoader(); + } + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void useCodeLoader(int count) + { + for(int i = 0; i < count; i++) + { + QCodeLoader qCodeLoader = QCodeLoader.getAdHoc(QCodeLoader.class, new QCodeReference(QCodeLoader.class)); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRendererTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRendererTest.java new file mode 100644 index 00000000..d8c70032 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRendererTest.java @@ -0,0 +1,96 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for Aggregate2DTableWidgetRenderer + *******************************************************************************/ +class Aggregate2DTableWidgetRendererTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of( + new QRecord().withValue("lastName", "Simpson").withValue("homeStateId", 50), + new QRecord().withValue("lastName", "Simpson").withValue("homeStateId", 50), + new QRecord().withValue("lastName", "Simpson").withValue("homeStateId", 50), + new QRecord().withValue("lastName", "Simpson").withValue("homeStateId", 49), + new QRecord().withValue("lastName", "Flanders").withValue("homeStateId", 49), + new QRecord().withValue("lastName", "Flanders").withValue("homeStateId", 49), + new QRecord().withValue("lastName", "Burns").withValue("homeStateId", 50) + ))); + + RenderWidgetInput input = new RenderWidgetInput(); + input.setWidgetMetaData(new QWidgetMetaData() + .withDefaultValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY) + .withDefaultValue("valueField", "id") + .withDefaultValue("rowField", "lastName") + .withDefaultValue("columnField", "homeStateId") + .withDefaultValue("orderBys", "row") + ); + RenderWidgetOutput output = new Aggregate2DTableWidgetRenderer().render(input); + TableData tableData = (TableData) output.getWidgetData(); + System.out.println(tableData.getRows()); + + TableDataAssert.assertThat(tableData) + .hasRowWithColumnContaining("_row", "Simpson", row -> + row.hasColumnContaining("50", "3") + .hasColumnContaining("49", "1") + .hasColumnContaining("_total", "4")) + .hasRowWithColumnContaining("_row", "Flanders", row -> + row.hasColumnContaining("50", "0") + .hasColumnContaining("49", "2") + .hasColumnContaining("_total", "2")) + .hasRowWithColumnContaining("_row", "Burns", row -> + row.hasColumnContaining("50", "1") + .hasColumnContaining("49", "0") + .hasColumnContaining("_total", "1")) + .hasRowWithColumnContaining("_row", "Total", row -> + row.hasColumnContaining("50", "4") + .hasColumnContaining("49", "3") + .hasColumnContaining("_total", "7")); + + List rowLabels = tableData.getRows().stream().map(r -> r.get("_row").toString()).toList(); + assertEquals(List.of("Burns", "Flanders", "Simpson", "Total"), rowLabels); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidgetTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidgetTest.java new file mode 100644 index 00000000..1dce9072 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidgetTest.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.dashboard.RenderWidgetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.AlertData; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for ProcessAlertWidget + *******************************************************************************/ +class ProcessAlertWidgetTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + MetaDataProducerHelper.processAllMetaDataProducersInPackage(QContext.getQInstance(), ProcessAlertWidget.class.getPackageName()); + + RenderWidgetInput input = new RenderWidgetInput(); + input.setWidgetMetaData(QContext.getQInstance().getWidget(ProcessAlertWidget.NAME)); + + /////////////////////////////////////////////////////////////////////////////////////////// + // make sure we run w/o exceptions (and w/ default outputs) if there are no query params // + /////////////////////////////////////////////////////////////////////////////////////////// + RenderWidgetOutput output = new RenderWidgetAction().execute(input); + assertEquals(AlertData.AlertType.WARNING, ((AlertData) output.getWidgetData()).getAlertType()); + assertEquals("Warning", ((AlertData) output.getWidgetData()).getHtml()); + + ////////////////////////////////////////////////////// + // make sure we input params come through to output // + ////////////////////////////////////////////////////// + input.addQueryParam("alertType", "ERROR"); + input.addQueryParam("alertHtml", "Do not touch Willy"); + output = new RenderWidgetAction().execute(input); + assertEquals(AlertData.AlertType.ERROR, ((AlertData) output.getWidgetData()).getAlertType()); + assertEquals("Do not touch Willy", ((AlertData) output.getWidgetData()).getHtml()); + + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/TableDataAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/TableDataAssert.java new file mode 100644 index 00000000..5724f815 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/TableDataAssert.java @@ -0,0 +1,215 @@ +/* + * Copyright © 2022-2023. ColdTrack . All Rights Reserved. + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Predicate; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; + + +/******************************************************************************* + ** AssertJ assert class for widget TableData + *******************************************************************************/ +public class TableDataAssert extends AbstractAssert +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected TableDataAssert(TableData actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static TableDataAssert assertThat(RenderWidgetOutput widgetOutput) + { + Assertions.assertThat(widgetOutput).isNotNull(); + QWidgetData widgetData = widgetOutput.getWidgetData(); + Assertions.assertThat(widgetData).isNotNull(); + Assertions.assertThat(widgetData).isInstanceOf(TableData.class); + return (new TableDataAssert((TableData) widgetData, TableDataAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static TableDataAssert assertThat(TableData actual) + { + return (new TableDataAssert(actual, TableDataAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert hasSize(int expectedSize) + { + Assertions.assertThat(actual.getRows()).hasSize(expectedSize); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert hasSizeAtLeast(int sizeAtLeast) + { + Assertions.assertThat(actual.getRows()).hasSizeGreaterThanOrEqualTo(sizeAtLeast); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert doesNotHaveRowWithColumnContaining(String columnName, String containingValue) + { + for(Map row : actual.getRows()) + { + if(row.containsKey(columnName)) + { + String value = String.valueOf(row.get(columnName)); + if(value != null && value.contains(containingValue)) + { + failWithMessage("Failed because a row was found with a value in the [" + columnName + "] column containing [" + containingValue + "]" + + (containingValue.equals(value) ? "" : " (full value: [" + value + "]).")); + } + } + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert hasRowWithColumnContaining(String columnName, String containingValue) + { + hasRowWithColumnContaining(columnName, containingValue, (row) -> + { + }); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert hasRowWithColumnContaining(String columnName, String containingValue, Consumer rowAsserter) + { + return hasRowWithColumnPredicate(columnName, value -> value != null && value.contains(containingValue), "containing [" + containingValue + "]", rowAsserter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert hasRowWithColumnMatching(String columnName, String matchingValue) + { + hasRowWithColumnMatching(columnName, matchingValue, (row) -> + { + }); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert hasRowWithColumnMatching(String columnName, String matchingValue, Consumer rowAsserter) + { + return hasRowWithColumnPredicate(columnName, value -> value != null && value.matches(matchingValue), "matching [" + matchingValue + "]", rowAsserter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert hasRowWithColumnEqualTo(String columnName, String equalToValue) + { + hasRowWithColumnEqualTo(columnName, equalToValue, (row) -> + { + }); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataAssert hasRowWithColumnEqualTo(String columnName, String equalToValue, Consumer rowAsserter) + { + return hasRowWithColumnPredicate(columnName, value -> Objects.equals(value, equalToValue), "equalTo [" + equalToValue + "]", rowAsserter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private TableDataAssert hasRowWithColumnPredicate(String columnName, Predicate predicate, String predicateDescription, Consumer rowAsserter) + { + List foundValuesInColumn = new ArrayList<>(); + for(Map row : actual.getRows()) + { + if(row.containsKey(columnName)) + { + String value = String.valueOf(row.get(columnName)); + foundValuesInColumn.add(value); + + if(predicate.test(value)) + { + TableDataRowAssert tableDataRowAssert = TableDataRowAssert.assertThat(row); + rowAsserter.accept(tableDataRowAssert); + + return (this); + } + } + } + + if(actual.getRows().isEmpty()) + { + failWithMessage("Failed because there are no rows in the table."); + } + else if(foundValuesInColumn.isEmpty()) + { + failWithMessage("Failed to find any rows with a column named: [" + columnName + "]"); + } + else + { + failWithMessage("Failed to find a row with column [" + columnName + "] " + predicateDescription + + ".\nFound values were:\n" + StringUtils.join("\n", foundValuesInColumn)); + } + return (null); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/TableDataRowAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/TableDataRowAssert.java new file mode 100644 index 00000000..83ecf3ad --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/TableDataRowAssert.java @@ -0,0 +1,192 @@ +/* + * Copyright © 2022-2023. ColdTrack . All Rights Reserved. + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** AssertJ assert class for a row of data from a widget TableData + *******************************************************************************/ +public class TableDataRowAssert extends AbstractAssert> +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected TableDataRowAssert(Map actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static TableDataRowAssert assertThat(Map actual) + { + return (new TableDataRowAssert(actual, TableDataRowAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataRowAssert hasColumnContaining(String columnName, String containingValue) + { + String value = String.valueOf(actual.get(columnName)); + Assertions.assertThat(value) + .withFailMessage("Expected column [" + columnName + "] in row [" + actual + "] to contain [" + containingValue + "], but it didn't") + .contains(containingValue); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataRowAssert hasNoSubRows() + { + Object subRowsObject = actual.get("subRows"); + if(subRowsObject != null) + { + @SuppressWarnings("unchecked") + List> subRowsList = (List>) subRowsObject; + if(!subRowsList.isEmpty()) + { + fail("Row [" + actual + "] should not have had any subRows, but it did."); + } + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataRowAssert hasSubRowWithColumnContaining(String columnName, String containingValue) + { + hasSubRowWithColumnContaining(columnName, containingValue, (row) -> + { + }); + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private TableDataRowAssert hasSubRowWithColumnPredicate(String columnName, Function predicate, String predicateDescription, Consumer rowAsserter) + { + Object subRowsObject = actual.get("subRows"); + Assertions.assertThat(subRowsObject) + .withFailMessage("subRows should not be null").isNotNull() + .withFailMessage("subRows should be a List").isInstanceOf(List.class); + + @SuppressWarnings("unchecked") + List> subRowsList = (List>) subRowsObject; + + List foundValuesInColumn = new ArrayList<>(); + for(Map row : subRowsList) + { + if(row.containsKey(columnName)) + { + String value = String.valueOf(row.get(columnName)); + foundValuesInColumn.add(value); + + if(value != null && predicate.apply(value)) + { + TableDataRowAssert tableDataRowAssert = TableDataRowAssert.assertThat(row); + rowAsserter.accept(tableDataRowAssert); + + return (this); + } + } + } + + if(foundValuesInColumn.isEmpty()) + { + failWithMessage("Failed to find any rows with a column named: [" + columnName + "]"); + } + else + { + failWithMessage("Failed to find a row with column [" + columnName + "] " + predicateDescription + + ".\nFound values were:\n" + StringUtils.join("\n", foundValuesInColumn)); + } + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataRowAssert hasSubRowWithColumnMatching(String columnName, String matchesValue, Consumer rowAsserter) + { + Function predicate = (value) -> ValueUtils.getValueAsString(value).matches(matchesValue); + return hasSubRowWithColumnPredicate(columnName, predicate, " matching [" + matchesValue + "]", rowAsserter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataRowAssert hasSubRowWithColumnContaining(String columnName, String containingValue, Consumer rowAsserter) + { + Function predicate = (value) -> ValueUtils.getValueAsString(value).contains(containingValue); + return hasSubRowWithColumnPredicate(columnName, predicate, " containing [" + containingValue + "]", rowAsserter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public TableDataRowAssert doesNotHaveSubRowWithColumnContaining(String columnName, String containingValue) + { + Object subRowsObject = actual.get("subRows"); + if(subRowsObject != null) + { + Assertions.assertThat(subRowsObject).withFailMessage("subRows should be a List").isInstanceOf(List.class); + + @SuppressWarnings("unchecked") + List> subRowsList = (List>) subRowsObject; + + for(Map row : subRowsList) + { + if(row.containsKey(columnName)) + { + String value = String.valueOf(row.get(columnName)); + if(value != null && value.contains(containingValue)) + { + failWithMessage("Failed because a row was found with a value in the [" + columnName + "] column containing [" + containingValue + "]" + + (containingValue.equals(value) ? "" : " (full value: [" + value + "]).")); + } + } + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessActionTest.java new file mode 100644 index 00000000..a26523a3 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessActionTest.java @@ -0,0 +1,139 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.processes; + + +import java.util.UUID; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for CancelProcessAction + *******************************************************************************/ +public class CancelProcessActionTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadInputs() + { + RunProcessInput input = new RunProcessInput(); + assertThatThrownBy(() -> new CancelProcessAction().execute(input)) + .hasMessageContaining("Process [null] is not defined"); + + input.setProcessName("foobar"); + assertThatThrownBy(() -> new CancelProcessAction().execute(input)) + .hasMessageContaining("Process [foobar] is not defined"); + + input.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE); + assertThatThrownBy(() -> new CancelProcessAction().execute(input)) + .hasMessageContaining("processUUID was not given"); + + input.setProcessUUID(UUID.randomUUID().toString()); + assertThatThrownBy(() -> new CancelProcessAction().execute(input)) + .hasMessageContaining("State for process UUID") + .hasMessageContaining("was not found"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + try + { + /////////////////////////////////////////////////////////////// + // start up the process - having it break upon frontend step // + /////////////////////////////////////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + input.setProcessUUID(runProcessOutput.getProcessUUID()); + + ///////////////////////////////////////////////////////////////////////////////// + // try to run the cancel action, but, with no cancel step, it should exit noop // + ///////////////////////////////////////////////////////////////////////////////// + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(CancelProcessAction.class); + new CancelProcessAction().execute(input); + assertThat(collectingLogger.getCollectedMessages()) + .anyMatch(m -> m.getMessage().contains("does not have a custom cancel step")); + collectingLogger.clear(); + + /////////////////////////////////////// + // add a cancel step to this process // + /////////////////////////////////////// + QContext.getQInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE) + .setCancelStep(new QBackendStepMetaData().withCode(new QCodeReference(CancelStep.class))); + + new CancelProcessAction().execute(input); + assertThat(collectingLogger.getCollectedMessages()) + .noneMatch(m -> m.getMessage().contains("does not have a custom cancel step")) + .anyMatch(m -> m.getMessage().contains("Running cancel step")); + assertEquals(1, CancelStep.callCount); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(CancelProcessAction.class); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class CancelStep implements BackendStep + { + static int callCount = 0; + + + + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + callCount++; + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java index f012677d..0caad01f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java @@ -27,13 +27,21 @@ import java.math.BigDecimal; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -53,7 +61,7 @@ public class RunBackendStepActionTest extends BaseTest { TestCallback callback = new TestCallback(); RunBackendStepInput request = new RunBackendStepInput(); - request.setProcessName("greet"); + request.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE); request.setStepName("prepare"); request.setCallback(callback); RunBackendStepOutput result = new RunBackendStepAction().execute(request); @@ -67,6 +75,60 @@ public class RunBackendStepActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMinMaxInputRecords() throws QException + { + //////////////////////////////////////////// + // put a min-input-records on the process // + //////////////////////////////////////////// + QContext.getQInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).withMinInputRecords(5); + + ////////////////////////////////////////////////////////////////////////////////////// + // insert fewer than that min - then run w/ non-filtered filter, and assert we fail // + ////////////////////////////////////////////////////////////////////////////////////// + for(int i = 0; i < 3; i++) + { + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord().withValue("firstName", String.valueOf(i)))); + } + + Supplier inputSupplier = () -> + { + RunBackendStepInput input = new RunBackendStepInput(); + input.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE); + input.setStepName("prepare"); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + return (input); + }; + + assertThatThrownBy(() -> new RunBackendStepAction().execute(inputSupplier.get())) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("Too few records"); + + //////////////////////////////////////////////////// + // insert a few more - and then it should succeed // + //////////////////////////////////////////////////// + for(int i = 3; i < 10; i++) + { + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord().withValue("firstName", String.valueOf(i)))); + } + + new RunBackendStepAction().execute(inputSupplier.get()); + + //////////////////////////////////////////////////////////// + // now put a max on the process, and it should fail again // + //////////////////////////////////////////////////////////// + QContext.getQInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).withMaxInputRecords(8); + + assertThatThrownBy(() -> new RunBackendStepAction().execute(inputSupplier.get())) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("Too many records"); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -100,19 +162,20 @@ public class RunBackendStepActionTest extends BaseTest for(QFieldMetaData field : fields) { rs.put(field.getName(), switch(field.getType()) - { - case STRING -> "ABC"; - case INTEGER -> 42; - case DECIMAL -> new BigDecimal("47"); - case BOOLEAN -> true; - case DATE, TIME, DATE_TIME -> null; - case TEXT -> """ - ABC - XYZ"""; - case HTML -> "Oh my"; - case PASSWORD -> "myPa**word"; - case BLOB -> new byte[] { 1, 2, 3, 4 }; - }); + { + case STRING -> "ABC"; + case INTEGER -> 42; + case LONG -> 42L; + case DECIMAL -> new BigDecimal("47"); + case BOOLEAN -> true; + case DATE, TIME, DATE_TIME -> null; + case TEXT -> """ + ABC + XYZ"""; + case HTML -> "Oh my"; + case PASSWORD -> "myPa**word"; + case BLOB -> new byte[] { 1, 2, 3, 4 }; + }); } return (rs); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessUpdateStepListTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessUpdateStepListTest.java new file mode 100644 index 00000000..7ae1ab99 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessUpdateStepListTest.java @@ -0,0 +1,211 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.processes; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessTest.NoopBackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.apache.commons.lang.BooleanUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RunProcessUpdateStepListTest extends BaseTest +{ + private static final String PROCESS_NAME = RunProcessUpdateStepListTest.class.getSimpleName(); + + private final static String STEP_START = "start"; + private final static String STEP_A = "a"; + private final static String STEP_B = "b"; + private final static String STEP_C = "c"; + private final static String STEP_1 = "1"; + private final static String STEP_2 = "2"; + private final static String STEP_3 = "3"; + private final static String STEP_END = "end"; + + private final static List LETTERS_STEP_LIST = List.of( + STEP_START, + STEP_A, + STEP_B, + STEP_C, + STEP_END + ); + + private final static List NUMBERS_STEP_LIST = List.of( + STEP_START, + STEP_1, + STEP_2, + STEP_3, + STEP_END + ); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGoingLettersPath() throws QException + { + QContext.getQInstance().addProcess(defineProcess()); + + //////////////////////////////////////////////////////////// + // start the process, telling it to go the "letters" path // + //////////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(PROCESS_NAME); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + runProcessInput.setValues(MapBuilder.of("which", "letters")); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that we got back the next-step name of A, and the updated list of frontend steps (A, C) // + //////////////////////////////////////////////////////////////////////////////////////////////////// + Optional nextStepName = runProcessOutput.getProcessState().getNextStepName(); + assertTrue(nextStepName.isPresent()); + assertEquals(STEP_A, nextStepName.get()); + assertEquals(List.of(STEP_A, STEP_C, STEP_END), runProcessOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList()); + + ///////////////////////////////////////////////// + // resume the process after that frontend step // + ///////////////////////////////////////////////// + runProcessInput.setProcessUUID(runProcessOutput.getProcessUUID()); + runProcessInput.setStartAfterStep(nextStepName.get()); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + + /////////////////////////////////////////////////////////////////////////////////////// + // assert we got back C as the next-step now, and no updated frontend list this time // + /////////////////////////////////////////////////////////////////////////////////////// + nextStepName = runProcessOutput.getProcessState().getNextStepName(); + assertTrue(nextStepName.isPresent()); + assertEquals(STEP_C, nextStepName.get()); + assertNull(runProcessOutput.getUpdatedFrontendStepList()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGoingNumbersPathAndSkippingAhead() throws QException + { + QContext.getQInstance().addProcess(defineProcess()); + + //////////////////////////////////////////////////////////////////////////////////// + // start the process, telling it to go the "numbers" path, and to skip ahead some // + //////////////////////////////////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(PROCESS_NAME); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + runProcessInput.setValues(MapBuilder.of("which", "numbers", "skipSomeSteps", true)); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that we got back the next-step name of 2, and the updated list of frontend steps (1, 3) // + //////////////////////////////////////////////////////////////////////////////////////////////////// + Optional nextStepName = runProcessOutput.getProcessState().getNextStepName(); + assertTrue(nextStepName.isPresent()); + assertEquals(STEP_END, nextStepName.get()); + assertEquals(List.of(STEP_2, STEP_END), runProcessOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcess() + { + QProcessMetaData process = new QProcessMetaData() + .withName(PROCESS_NAME) + .withStepList(List.of( + new QBackendStepMetaData() + .withName(STEP_START) + .withCode(new QCodeReference(StartStep.class)), + new QFrontendStepMetaData() + .withName(STEP_END) + )); + + process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_A)); + process.addOptionalStep(new QBackendStepMetaData().withName(STEP_B).withCode(new QCodeReference(NoopBackendStep.class))); + process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_C)); + + process.addOptionalStep(new QBackendStepMetaData().withName(STEP_1).withCode(new QCodeReference(NoopBackendStep.class))); + process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_2)); + process.addOptionalStep(new QBackendStepMetaData().withName(STEP_3).withCode(new QCodeReference(NoopBackendStep.class))); + + return (process); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class StartStep implements BackendStep + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + boolean skipSomeSteps = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean("skipSomeSteps")); + + if(runBackendStepInput.getValueString("which").equals("letters")) + { + runBackendStepOutput.updateStepList(LETTERS_STEP_LIST); + if(skipSomeSteps) + { + runBackendStepOutput.setOverrideLastStepName(STEP_C); + } + } + else + { + runBackendStepOutput.updateStepList(NUMBERS_STEP_LIST); + if(skipSomeSteps) + { + runBackendStepOutput.setOverrideLastStepName(STEP_2); + } + } + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java index 9ff5f7de..ff089474 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -118,6 +120,26 @@ class ExportActionTest extends BaseTest runReport(recordCount, filename, ReportFormat.XLSX, true); File file = new File(filename); + LocalMacDevUtils.openFile(file.getAbsolutePath()); + + assertTrue(file.delete()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testExcelPOI() throws Exception + { + int recordCount = 1000; + String filename = "/tmp/ReportActionTest-POI.xlsx"; + + runReport(recordCount, filename, ReportFormat.XLSX, true); + + File file = new File(filename); + LocalMacDevUtils.openFile(file.getAbsolutePath()); assertTrue(file.delete()); } @@ -147,9 +169,10 @@ class ExportActionTest extends BaseTest ExportInput exportInput = new ExportInput(); exportInput.setTableName(TestUtils.TABLE_NAME_ORDER); - exportInput.setReportFormat(ReportFormat.CSV); ByteArrayOutputStream reportOutputStream = new ByteArrayOutputStream(); - exportInput.setReportOutputStream(reportOutputStream); + exportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.CSV) + .withReportOutputStream(reportOutputStream)); exportInput.setQueryFilter(new QQueryFilter()); exportInput.setFieldNames(List.of("id", "orderNo", "storeId", "orderLine.id", "orderLine.sku", "orderLine.quantity")); // exportInput.setFieldNames(List.of("id", "orderNo", "storeId")); @@ -197,8 +220,7 @@ class ExportActionTest extends BaseTest exportInput.setTableName("person"); QTableMetaData table = exportInput.getTable(); - exportInput.setReportFormat(reportFormat); - exportInput.setReportOutputStream(outputStream); + exportInput.setReportDestination(new ReportDestination().withReportFormat(reportFormat).withReportOutputStream(outputStream)); exportInput.setQueryFilter(new QQueryFilter()); exportInput.setLimit(recordCount); @@ -243,7 +265,7 @@ class ExportActionTest extends BaseTest /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // use xlsx, which has a max-rows limit, to verify that code runs, but doesn't throw when there aren't too many rows // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - exportInput.setReportFormat(ReportFormat.XLSX); + exportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.XLSX)); new ExportAction().preExecute(exportInput); @@ -278,7 +300,7 @@ class ExportActionTest extends BaseTest //////////////////////////////////////////////////////////////// // use xlsx, which has a max-cols limit, to verify that code. // //////////////////////////////////////////////////////////////// - exportInput.setReportFormat(ReportFormat.XLSX); + exportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.XLSX)); assertThrows(QUserFacingException.class, () -> { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java index 590d96e6..5f354631 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java @@ -23,19 +23,33 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.Serializable; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.Month; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.BoldHeaderAndFooterPoiExcelStyler; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.ExcelPoiBasedStreamingExportStreamer; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.PoiExcelStylerInterface; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; @@ -51,7 +65,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.testutils.PersonQRecord; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFPivotTable; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -85,14 +109,14 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot1() throws QException + void testSummary1() throws QException { QInstance qInstance = QContext.getQInstance(); - qInstance.addReport(definePersonShoesPivotReport(true)); + qInstance.addReport(definePersonShoesSummaryReport(true)); insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1), "endDate", LocalDate.of(1980, Month.DECEMBER, 31))); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(3, list.size()); @@ -140,10 +164,10 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot2() throws QException + void testSummary2() throws QException { QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = definePersonShoesPivotReport(false); + QReportMetaData report = definePersonShoesSummaryReport(false); ////////////////////////////////////////////// // change from the default to sort reversed // @@ -153,7 +177,7 @@ public class GenerateReportActionTest extends BaseTest insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1), "endDate", LocalDate.of(1980, Month.DECEMBER, 31))); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(2, list.size()); @@ -172,10 +196,10 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot3() throws QException + void testSummary3() throws QException { QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = definePersonShoesPivotReport(false); + QReportMetaData report = definePersonShoesSummaryReport(false); ////////////////////////////////////////////////////////////////////////////////////////////// // remove the filters, change to sort by personCount (to get some ties), then sumPrice desc // @@ -187,7 +211,7 @@ public class GenerateReportActionTest extends BaseTest insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.now(), "endDate", LocalDate.now())); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); @@ -224,16 +248,16 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot4() throws QException + void testSummary4() throws QException { QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = definePersonShoesPivotReport(false); + QReportMetaData report = definePersonShoesSummaryReport(false); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // remove the filter, change to have 2 pivot columns - homeStateId and lastName - we should get no roll-up like this. // //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// report.getDataSources().get(0).getQueryFilter().setCriteria(null); - report.getViews().get(0).setPivotFields(List.of( + report.getViews().get(0).setSummaryFields(List.of( "homeStateId", "lastName" )); @@ -241,7 +265,7 @@ public class GenerateReportActionTest extends BaseTest insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.now(), "endDate", LocalDate.now())); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(6, list.size()); @@ -282,21 +306,21 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot5() throws QException + void testSummary5() throws QException { QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = definePersonShoesPivotReport(false); + QReportMetaData report = definePersonShoesSummaryReport(false); ///////////////////////////////////////////////////////////////////////////////////// // remove the filter, and just pivot on homeStateId - should aggregate differently // ///////////////////////////////////////////////////////////////////////////////////// report.getDataSources().get(0).getQueryFilter().setCriteria(null); - report.getViews().get(0).setPivotFields(List.of("homeStateId")); + report.getViews().get(0).setSummaryFields(List.of("homeStateId")); qInstance.addReport(report); insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.now(), "endDate", LocalDate.now())); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(2, list.size()); @@ -315,23 +339,18 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - @Test - void runToCsv() throws Exception + private String runToString(ReportFormat reportFormat, String reportName) throws Exception { - String name = "/tmp/report.csv"; + String name = "/tmp/report." + reportFormat.getExtension(); try(FileOutputStream fileOutputStream = new FileOutputStream(name)) { - QInstance qInstance = QContext.getQInstance(); - qInstance.addReport(definePersonShoesPivotReport(true)); - insertPersonRecords(qInstance); - ReportInput reportInput = new ReportInput(); - reportInput.setReportName(REPORT_NAME); - reportInput.setReportFormat(ReportFormat.CSV); - reportInput.setReportOutputStream(fileOutputStream); + reportInput.setReportName(reportName); + reportInput.setReportDestination(new ReportDestination().withReportFormat(reportFormat).withReportOutputStream(fileOutputStream)); reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); new GenerateReportAction().execute(reportInput); System.out.println("Wrote File: " + name); + return (FileUtils.readFileToString(new File(name), StandardCharsets.UTF_8)); } } @@ -341,27 +360,189 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void runToXlsx() throws Exception + void runSummaryToXlsx() throws Exception { - String name = "/tmp/report.xlsx"; + ReportFormat format = ReportFormat.XLSX; + String name = "/tmp/report-" + format + ".xlsx"; try(FileOutputStream fileOutputStream = new FileOutputStream(name)) { QInstance qInstance = QContext.getQInstance(); - qInstance.addReport(definePersonShoesPivotReport(true)); + qInstance.addReport(definePersonShoesSummaryReport(true)); insertPersonRecords(qInstance); ReportInput reportInput = new ReportInput(); reportInput.setReportName(REPORT_NAME); - reportInput.setReportFormat(ReportFormat.XLSX); - reportInput.setReportOutputStream(fileOutputStream); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream)); reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); new GenerateReportAction().execute(reportInput); System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name); } } + /******************************************************************************* + ** Keep some test coverage on the fastexcel library (as long as we keep it around) + *******************************************************************************/ + @Test + void runTableToXlsxFastexcel() throws Exception + { + ReportFormat format = ReportFormat.XLSX; + String name = "/tmp/report-fastexcel.xlsx"; + try(FileOutputStream fileOutputStream = new FileOutputStream(name)) + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(defineTableOnlyReport()); + insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + reportInput.setOverrideExportStreamerSupplier(ExcelFastexcelExportStreamer::new); + new GenerateReportAction().execute(reportInput); + System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name); + } + } + + + + /******************************************************************************* + ** Imagine if we used the boldHeaderAndFooter styler + *******************************************************************************/ + @Test + void runTableToXlsxWithOverrideStyles() throws Exception + { + ReportFormat format = ReportFormat.XLSX; + String name = "/tmp/report-fastexcel.xlsx"; + try(FileOutputStream fileOutputStream = new FileOutputStream(name)) + { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData reportMetaData = defineTableOnlyReport(); + reportMetaData.getViews().get(0).withTitleFormat("My Title"); + + qInstance.addReport(reportMetaData); + insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + + reportInput.setOverrideExportStreamerSupplier(() -> new ExcelPoiBasedStreamingExportStreamer() + { + @Override + protected PoiExcelStylerInterface getStylerInterface() + { + return new BoldHeaderAndFooterPoiExcelStyler(); + } + }); + + new GenerateReportAction().execute(reportInput); + System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void runTableToXlsx() throws Exception + { + ReportFormat format = ReportFormat.XLSX; + String name = "/tmp/report-" + format + ".xlsx"; + try(FileOutputStream fileOutputStream = new FileOutputStream(name)) + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(defineTableOnlyReport()); + insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + new GenerateReportAction().execute(reportInput); + System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void runPivotToXlsx() throws Exception + { + String name = "/tmp/pivot-test.xlsx"; + try(FileOutputStream fileOutputStream = new FileOutputStream(name)) + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(definePivotReport()); + insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.XLSX).withReportOutputStream(fileOutputStream)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + new GenerateReportAction().execute(reportInput); + System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name, "/Applications/Numbers.app"); + } + + /////////////////////////////////////////////////////////// + // read the file we wrote, and assert about its contents // + /////////////////////////////////////////////////////////// + FileInputStream file = new FileInputStream(name); + XSSFWorkbook workbook = new XSSFWorkbook(file); + + XSSFSheet sheet = workbook.getSheetAt(1); + List pivotTables = sheet.getPivotTables(); + XSSFPivotTable xssfPivotTable = pivotTables.get(0); + List rowLabelColumns = xssfPivotTable.getRowLabelColumns(); + List colLabelColumns = xssfPivotTable.getColLabelColumns(); + Sheet dataSheet = xssfPivotTable.getDataSheet(); + Sheet parentSheet = xssfPivotTable.getParentSheet(); + System.out.println(); + + Map> data = new HashMap<>(); + int i = 0; + for(Row row : sheet) + { + data.put(i, new ArrayList<>()); + + for(Cell cell : row) + { + data.get(i).add(switch(cell.getCellType()) + { + case _NONE -> "<_NONE>"; + case NUMERIC -> cell.getNumericCellValue(); + case STRING -> cell.getStringCellValue(); + case FORMULA -> cell.getCellFormula(); + case BLANK -> ""; + case BOOLEAN -> cell.getBooleanCellValue(); + case ERROR -> cell.getErrorCellValue(); + }); + } + i++; + } + + System.out.println(data); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -369,8 +550,7 @@ public class GenerateReportActionTest extends BaseTest { ReportInput reportInput = new ReportInput(); reportInput.setReportName(REPORT_NAME); - reportInput.setReportFormat(ReportFormat.LIST_OF_MAPS); - reportInput.setReportOutputStream(new ByteArrayOutputStream()); + reportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.LIST_OF_MAPS).withReportOutputStream(new ByteArrayOutputStream())); reportInput.setInputValues(inputValues); new GenerateReportAction().execute(reportInput); } @@ -380,15 +560,15 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private void insertPersonRecords(QInstance qInstance) throws QException + public static void insertPersonRecords(QInstance qInstance) throws QException { TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of( - new PersonQRecord().withLastName("Jonson").withBirthDate(LocalDate.of(1980, Month.JANUARY, 31)).withNoOfShoes(null).withHomeStateId(1).withPrice(null).withCost(new BigDecimal("0.50")), // wrong last initial - new PersonQRecord().withLastName("Jones").withBirthDate(LocalDate.of(1980, Month.JANUARY, 31)).withNoOfShoes(3).withHomeStateId(1).withPrice(new BigDecimal("1.00")).withCost(new BigDecimal("0.50")), // wrong last initial - new PersonQRecord().withLastName("Kelly").withBirthDate(LocalDate.of(1979, Month.DECEMBER, 30)).withNoOfShoes(4).withHomeStateId(1).withPrice(new BigDecimal("1.20")).withCost(new BigDecimal("0.50")), // bad birthdate - new PersonQRecord().withLastName("Keller").withBirthDate(LocalDate.of(1980, Month.JANUARY, 7)).withNoOfShoes(5).withHomeStateId(1).withPrice(new BigDecimal("2.40")).withCost(new BigDecimal("3.50")), - new PersonQRecord().withLastName("Kelkhoff").withBirthDate(LocalDate.of(1980, Month.FEBRUARY, 15)).withNoOfShoes(6).withHomeStateId(1).withPrice(new BigDecimal("3.60")).withCost(new BigDecimal("3.50")), - new PersonQRecord().withLastName("Kelkhoff").withBirthDate(LocalDate.of(1980, Month.MARCH, 20)).withNoOfShoes(7).withHomeStateId(2).withPrice(new BigDecimal("4.80")).withCost(new BigDecimal("3.50")) + new PersonQRecord().withFirstName("Darin").withLastName("Jonson").withBirthDate(LocalDate.of(1980, Month.JANUARY, 31)).withNoOfShoes(null).withHomeStateId(1).withPrice(null).withCost(new BigDecimal("0.50")), // wrong last initial + new PersonQRecord().withFirstName("Darin").withLastName("Jones").withBirthDate(LocalDate.of(1980, Month.JANUARY, 31)).withNoOfShoes(3).withHomeStateId(1).withPrice(new BigDecimal("1.00")).withCost(new BigDecimal("0.50")), // wrong last initial + new PersonQRecord().withFirstName("Darin").withLastName("Kelly").withBirthDate(LocalDate.of(1979, Month.DECEMBER, 30)).withNoOfShoes(4).withHomeStateId(1).withPrice(new BigDecimal("1.20")).withCost(new BigDecimal("0.50")), // bad birthdate + new PersonQRecord().withFirstName("Trevor").withLastName("Keller").withBirthDate(LocalDate.of(1980, Month.JANUARY, 7)).withNoOfShoes(5).withHomeStateId(1).withPrice(new BigDecimal("2.40")).withCost(new BigDecimal("3.50")), + new PersonQRecord().withFirstName("Trevor").withLastName("Kelkhoff").withBirthDate(LocalDate.of(1980, Month.FEBRUARY, 15)).withNoOfShoes(6).withHomeStateId(1).withPrice(new BigDecimal("3.60")).withCost(new BigDecimal("3.50")), + new PersonQRecord().withFirstName("Kelly").withLastName("Kelkhoff").withBirthDate(LocalDate.of(1980, Month.MARCH, 20)).withNoOfShoes(7).withHomeStateId(2).withPrice(new BigDecimal("4.80")).withCost(new BigDecimal("3.50")) )); } @@ -397,7 +577,7 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - public static QReportMetaData definePersonShoesPivotReport(boolean includeTotalRow) + public static QReportMetaData definePersonShoesSummaryReport(boolean includeTotalRow) { return new QReportMetaData() .withName(REPORT_NAME) @@ -416,11 +596,11 @@ public class GenerateReportActionTest extends BaseTest )) .withViews(List.of( new QReportView() - .withName("pivot") - .withLabel("pivot") + .withName("summary") + .withLabel("summary") .withDataSourceName("persons") .withType(ReportType.SUMMARY) - .withPivotFields(List.of("lastName")) + .withSummaryFields(List.of("lastName")) .withIncludeTotalRow(includeTotalRow) .withTitleFormat("Number of shoes - people born between %s and %s - pivot on LastName, sort by Quantity, Revenue DESC") .withTitleFields(List.of("${input.startDate}", "${input.endDate}")) @@ -452,39 +632,14 @@ public class GenerateReportActionTest extends BaseTest @Test void testTableOnlyReport() throws QException { - QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = new QReportMetaData() - .withName(REPORT_NAME) - .withDataSources(List.of( - new QReportDataSource() - .withName("persons") - .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) - .withQueryFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of("${input.startDate}"))) - ) - )) - .withInputFields(List.of( - new QFieldMetaData("startDate", QFieldType.DATE_TIME) - )) - .withViews(List.of( - new QReportView() - .withName("table1") - .withLabel("table1") - .withDataSourceName("persons") - .withType(ReportType.TABLE) - .withColumns(List.of( - new QReportField().withName("id"), - new QReportField().withName("firstName"), - new QReportField().withName("lastName") - )) - )); - + QInstance qInstance = QContext.getQInstance(); + QReportMetaData report = defineTableOnlyReport(); qInstance.addReport(report); insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1))); - List> list = ListOfMapsExportStreamer.getList("table1"); + List> list = ListOfMapsExportStreamer.getList("Table 1"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(5, list.size()); @@ -493,6 +648,88 @@ public class GenerateReportActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportMetaData defineTableOnlyReport() + { + QReportMetaData report = new QReportMetaData() + .withName(REPORT_NAME) + .withDataSources(List.of( + new QReportDataSource() + .withName("persons") + .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of("${input.startDate}")))))) + + .withInputFields(List.of( + new QFieldMetaData("startDate", QFieldType.DATE_TIME))) + + .withViews(List.of( + new QReportView() + .withName("table1") + .withLabel("Table 1") + .withDataSourceName("persons") + .withType(ReportType.TABLE) + .withColumns(List.of( + new QReportField().withName("id"), + new QReportField().withName("firstName"), + new QReportField().withName("lastName"))))); + + return report; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportMetaData definePivotReport() + { + QReportMetaData report = new QReportMetaData() + .withName(REPORT_NAME) + .withDataSources(List.of( + new QReportDataSource() + .withName("persons") + .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of("${input.startDate}")))))) + + .withInputFields(List.of( + new QFieldMetaData("startDate", QFieldType.DATE_TIME))) + + .withViews(List.of( + + new QReportView() + .withName("table1") + .withLabel("Table 1") + .withDataSourceName("persons") + .withType(ReportType.TABLE) + .withColumns(List.of( + new QReportField().withName("id"), + new QReportField().withName("firstName"), + new QReportField().withName("lastName"), + new QReportField().withName("homeStateId") + )), + + new QReportView() + .withName("pivotTable1") + .withLabel("My Pivot Table") + + .withType(ReportType.PIVOT) + .withPivotTableSourceViewName("table1") + .withPivotTableDefinition(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("homeStateId")) + // .withRow(new PivotTableGroupBy().withFieldName("lastName")) + // .withColumn(new PivotTableGroupBy().withFieldName("firstName")) + .withValue(new PivotTableValue().withFunction(PivotTableFunction.COUNT).withFieldName("id"))) + )); + + return report; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -500,6 +737,180 @@ public class GenerateReportActionTest extends BaseTest void testTwoTableViewsOneDataSourceReport() throws QException { QInstance qInstance = QContext.getQInstance(); + defineTwoViewsOneDataSourceReport(qInstance); + + insertPersonRecords(qInstance); + runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1))); + + List> list = ListOfMapsExportStreamer.getList("Table 1"); + Iterator> iterator = list.iterator(); + Map row = iterator.next(); + assertEquals(5, list.size()); + assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name"); + + list = ListOfMapsExportStreamer.getList("Table 2"); + iterator = list.iterator(); + row = iterator.next(); + assertEquals(5, list.size()); + assertThat(row).containsOnlyKeys("Birth Date"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneTableViewsOneDataSourceJsonReport() throws Exception + { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData report = defineTableOnlyReport(); + qInstance.addReport(report); + + insertPersonRecords(qInstance); + String json = runToString(ReportFormat.JSON, report.getName()); + // System.out.println(json); + + ///////////////////////////////////////////////////////////////////////////////// + // for a one-view report, we should just have an array of the report's records // + ///////////////////////////////////////////////////////////////////////////////// + JSONArray jsonArray = new JSONArray(json); + assertEquals(6, jsonArray.length()); + assertThat(jsonArray.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("lastName", "Jonson"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTwoTableViewsOneDataSourceJsonReport() throws Exception + { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData report = defineTwoViewsOneDataSourceReport(qInstance); + + insertPersonRecords(qInstance); + String json = runToString(ReportFormat.JSON, report.getName()); + // System.out.println(json); + + ///////////////////////////////////////////////////////////////////////////////// + // for a multi-view report, we should have an array with the views as elements // + ///////////////////////////////////////////////////////////////////////////////// + JSONArray jsonArray = new JSONArray(json); + assertEquals(2, jsonArray.length()); + + JSONObject firstView = jsonArray.getJSONObject(0); + assertEquals("Table 1", firstView.getString("name")); + JSONArray firstViewData = firstView.getJSONArray("data"); + assertEquals(6, firstViewData.length()); + assertThat(firstViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("lastName", "Jonson"); + + JSONObject secondView = jsonArray.getJSONObject(1); + assertEquals("Table 2", secondView.getString("name")); + JSONArray secondViewData = secondView.getJSONArray("data"); + assertEquals(6, secondViewData.length()); + assertThat(secondViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("birthDate", "1980-01-31"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableViewsAndSummaryViewJsonReport() throws Exception + { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData report = defineSimplePersonTableAndSummaryByFirstNameReport(); + qInstance.addReport(report); + + insertPersonRecords(qInstance); + String json = runToString(ReportFormat.JSON, report.getName()); + System.out.println(json); + + ///////////////////////////////////////////////////////////////////////////////// + // for a multi-view report, we should have an array with the views as elements // + ///////////////////////////////////////////////////////////////////////////////// + JSONArray jsonArray = new JSONArray(json); + assertEquals(2, jsonArray.length()); + + JSONObject firstView = jsonArray.getJSONObject(0); + assertEquals("Table 1", firstView.getString("name")); + JSONArray firstViewData = firstView.getJSONArray("data"); + assertEquals(6, firstViewData.length()); + assertThat(firstViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("lastName", "Jonson"); + + JSONObject secondView = jsonArray.getJSONObject(1); + assertEquals("Summary", secondView.getString("name")); + JSONArray secondViewData = secondView.getJSONArray("data"); + assertEquals(4, secondViewData.length()); + assertThat(secondViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("personCount", 3); + assertThat(secondViewData.getJSONObject(3).toMap()) + .hasFieldOrPropertyWithValue("firstName", "Totals") + .hasFieldOrPropertyWithValue("personCount", 6); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QReportMetaData defineSimplePersonTableAndSummaryByFirstNameReport() + { + return new QReportMetaData() + .withName(REPORT_NAME) + .withDataSources(List.of( + new QReportDataSource() + .withName("persons") + .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilter(new QQueryFilter()) + )) + .withViews(List.of( + new QReportView() + .withName("table1") + .withLabel("Table 1") + .withDataSourceName("persons") + .withType(ReportType.TABLE) + .withColumns(List.of( + new QReportField().withName("id"), + new QReportField().withName("firstName"), + new QReportField().withName("lastName"), + new QReportField().withName("homeStateId") + )), + new QReportView() + .withName("summary") + .withLabel("Summary") + .withDataSourceName("persons") + .withType(ReportType.SUMMARY) + .withSummaryFields(List.of("firstName")) + .withIncludeTotalRow(true) + .withOrderByFields(List.of(new QFilterOrderBy("personCount", false))) + .withColumns(List.of( + new QReportField().withName("personCount").withLabel("Person Count").withFormula("${pivot.count.id}").withDisplayFormat(DisplayFormat.COMMAS) + )) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportMetaData defineTwoViewsOneDataSourceReport(QInstance qInstance) + { QReportMetaData report = new QReportMetaData() .withName(REPORT_NAME) .withDataSources(List.of( @@ -516,7 +927,7 @@ public class GenerateReportActionTest extends BaseTest .withViews(List.of( new QReportView() .withName("table1") - .withLabel("table1") + .withLabel("Table 1") .withDataSourceName("persons") .withType(ReportType.TABLE) .withColumns(List.of( @@ -526,7 +937,7 @@ public class GenerateReportActionTest extends BaseTest )), new QReportView() .withName("table2") - .withLabel("table2") + .withLabel("Table 2") .withDataSourceName("persons") .withType(ReportType.TABLE) .withColumns(List.of( @@ -535,21 +946,7 @@ public class GenerateReportActionTest extends BaseTest )); qInstance.addReport(report); - - insertPersonRecords(qInstance); - runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1))); - - List> list = ListOfMapsExportStreamer.getList("table1"); - Iterator> iterator = list.iterator(); - Map row = iterator.next(); - assertEquals(5, list.size()); - assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name"); - - list = ListOfMapsExportStreamer.getList("table2"); - iterator = list.iterator(); - row = iterator.next(); - assertEquals(5, list.size()); - assertThat(row).containsOnlyKeys("Birth Date"); + return (report); } @@ -566,8 +963,7 @@ public class GenerateReportActionTest extends BaseTest ReportInput reportInput = new ReportInput(); reportInput.setReportName(TestUtils.REPORT_NAME_PERSON_SIMPLE); - reportInput.setReportFormat(ReportFormat.LIST_OF_MAPS); - reportInput.setReportOutputStream(new ByteArrayOutputStream()); + reportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.LIST_OF_MAPS).withReportOutputStream(new ByteArrayOutputStream())); new GenerateReportAction().execute(reportInput); List> list = ListOfMapsExportStreamer.getList("Simple Report"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java new file mode 100644 index 00000000..126272bc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java @@ -0,0 +1,54 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for JsonExportStreamer + *******************************************************************************/ +class JsonExportStreamerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + Function runOne = label -> new JsonExportStreamer().getLabelForJson(new QFieldMetaData("test", QFieldType.STRING).withLabel(label)); + assertEquals("sku", runOne.apply("SKU")); + assertEquals("clientName", runOne.apply("Client Name")); + assertEquals("slaStatus", runOne.apply("SLA Status")); + assertEquals("lineItem:sku", runOne.apply("Line Item: SKU")); + assertEquals("parcel:slaStatus", runOne.apply("Parcel: SLA Status")); + assertEquals("order:client", runOne.apply("Order: Client")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java index e17d6ed0..bbb5493b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java @@ -26,15 +26,21 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; @@ -777,4 +783,142 @@ class InsertActionTest extends BaseTest assertEquals(2, records.get(1).getValueInteger("noOfShoes")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCustomizers() throws QException + { + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(TestPreInsertCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"))) + .getRecords(); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // because this was a pre-action, the value should actually be inserted - so re-query and get it // + /////////////////////////////////////////////////////////////////////////////////////////////////// + assertEquals(1701, new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(1)).getValueInteger("noOfShoes")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(TestPostInsertCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Thom").withValue("lastName", "Chutterloin"))) + .getRecords(); + assertEquals(47, records.get(0).getValueInteger("homeStateId")); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // because this was a post-action, the value should NOT actually be inserted - so re-query and confirm null // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertNull(new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(2)).getValueInteger("homeStateId")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(TestTableCustomizer.class)); + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(TestTableCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Thom").withValue("lastName", "Chutterloin"))) + .getRecords(); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + assertEquals(47, records.get(0).getValueInteger("homeStateId")); + + ////////////////////////////////////////////////////////////////////// + // merger of the two above - one pre, one post, so one set, one not // + ////////////////////////////////////////////////////////////////////// + QRecord fetchedRecord = new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(2)); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + assertNull(fetchedRecord.getValueInteger("homeStateId")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestPreInsertCustomizer extends AbstractPreInsertCustomizer + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) throws QException + { + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("noOfShoes", 1701))); + return rs; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestPostInsertCustomizer extends AbstractPostInsertCustomizer + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // grr, memory backend let's make sure to return a clone (so we don't edit what's stored!) // + ///////////////////////////////////////////////////////////////////////////////////////////// + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("homeStateId", 47))); + return rs; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTableCustomizer implements TableCustomizerInterface + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("noOfShoes", 1701))); + return rs; + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // grr, memory backend let's make sure to return a clone (so we don't edit what's stored!) // + ///////////////////////////////////////////////////////////////////////////////////////////// + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("homeStateId", 47))); + return rs; + } + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QQueryFilterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QQueryFilterTest.java new file mode 100644 index 00000000..1257d7e2 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QQueryFilterTest.java @@ -0,0 +1,143 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.tables; + + +import java.io.Serializable; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.FilterVariableExpression; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.BETWEEN; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IS_BLANK; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for QQueryFilter + ** + *******************************************************************************/ +class QQueryFilterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInterpretValues() throws QException + { + Map inputValues = new HashMap<>(); + inputValues.put("clientIdEquals1", "value"); + + AbstractFilterExpression expression = new FilterVariableExpression() + .withVariableName("clientIdEquals1"); + + QQueryFilter qQueryFilter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression)); + qQueryFilter.interpretValues(inputValues); + + assertEquals("value", qQueryFilter.getCriteria().get(0).getValues().get(0)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInterpretValuesNotInMap() throws QException + { + AbstractFilterExpression expression = new FilterVariableExpression() + .withVariableName("clientIdEquals1"); + + QQueryFilter qQueryFilter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression)); + assertThatThrownBy(() -> qQueryFilter.interpretValues(Collections.emptyMap())) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("Missing value for variable: clientIdEquals1"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInterpretValuesEmptyString() throws QException + { + Map inputValues = new HashMap<>(); + inputValues.put("clientIdEquals1", ""); + + AbstractFilterExpression expression = new FilterVariableExpression() + .withVariableName("clientIdEquals1"); + + QQueryFilter qQueryFilter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression)); + assertThatThrownBy(() -> qQueryFilter.interpretValues(inputValues)) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("Missing value for variable: clientIdEquals1"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testPrepForBackend() throws QException + { + FilterVariableExpression fve0 = new FilterVariableExpression(); + FilterVariableExpression fve1 = new FilterVariableExpression(); + FilterVariableExpression fve2 = new FilterVariableExpression(); + FilterVariableExpression fve3 = new FilterVariableExpression(); + FilterVariableExpression fve4 = new FilterVariableExpression(); + FilterVariableExpression fve5 = new FilterVariableExpression(); + FilterVariableExpression fve6 = new FilterVariableExpression(); + FilterVariableExpression fve7 = new FilterVariableExpression(); + + QQueryFilter qQueryFilter = new QQueryFilter( + new QFilterCriteria("id", EQUALS, fve0), + new QFilterCriteria("value", IS_BLANK, fve1), + new QFilterCriteria("id", EQUALS, fve2), + new QFilterCriteria("id", BETWEEN, fve3, fve4), + new QFilterCriteria("id", BETWEEN, fve5, fve6), + new QFilterCriteria("joinTable.someFieldId", EQUALS, fve7) + ); + qQueryFilter.prepForBackend(); + + assertEquals("idEquals", fve0.getVariableName()); + assertEquals("valueIsBlank", fve1.getVariableName()); + assertEquals("idEquals2", fve2.getVariableName()); + assertEquals("idBetweenFrom", fve3.getVariableName()); + assertEquals("idBetweenTo", fve4.getVariableName()); + assertEquals("idBetweenFrom2", fve5.getVariableName()); + assertEquals("idBetweenTo2", fve6.getVariableName()); + assertEquals("joinTableSomeFieldIdEquals", fve7.getVariableName()); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java index 4ce0e8ee..9eb5dc57 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.tables; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; @@ -38,12 +39,15 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* - ** Unit test for ReplaceAction + ** Unit test for ReplaceAction *******************************************************************************/ class ReplaceActionTest extends BaseTest { @@ -81,12 +85,27 @@ class ReplaceActionTest extends BaseTest replaceInput.setOmitDmlAudit(true); replaceInput.setRecords(newPeople); replaceInput.setFilter(null); + replaceInput.setSetPrimaryKeyInInsertedRecords(false); ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); assertEquals(1, replaceOutput.getInsertOutput().getRecords().size()); assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size()); assertEquals(1, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // due to false for SetPrimaryKeyInInsertedRecords, make sure primary keys aren't on the records that got inserted // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Optional ned = newPeople.stream().filter(r -> r.getValueString("firstName").equals("Ned")).findFirst(); + assertThat(ned).isPresent(); + assertNull(ned.get().getValue("id"), "the record that got inserted should not have its primary key set"); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // but note, homer (who was updated) would have had its primary key set too, as part of the internal processing that does the update. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Optional homer = newPeople.stream().filter(r -> r.getValueString("firstName").equals("Homer")).findFirst(); + assertThat(homer).isPresent(); + assertNotNull(homer.get().getValue("id"), "the record that got updated should have its primary key set"); + ////////////////////////////// // assert homer was updated // ////////////////////////////// @@ -136,12 +155,18 @@ class ReplaceActionTest extends BaseTest replaceInput.setOmitDmlAudit(true); replaceInput.setRecords(newPeople); replaceInput.setFilter(null); + replaceInput.setSetPrimaryKeyInInsertedRecords(true); ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); assertEquals(2, replaceOutput.getInsertOutput().getRecords().size()); assertEquals(0, replaceOutput.getUpdateOutput().getRecords().size()); assertEquals(2, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // due to true for SetPrimaryKeyInInsertedRecords, make sure primary keys ARE on all the records that got inserted // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertTrue(newPeople.stream().allMatch(r -> r.getValue("id") != null), "All inserted records should have their primary key"); + /////////////////////////////////////// // assert homer & marge were deleted // /////////////////////////////////////// @@ -157,6 +182,134 @@ class ReplaceActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTwoKeysWithNullsNotMatchingAllowingDelete() throws QException + { + String tableName = TestUtils.TABLE_NAME_TWO_KEYS; + + //////////////////////////////// + // start with these 2 records // + //////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ))); + + //////////////////////////////////////////////////// + // now do a replace action that just updates them // + //////////////////////////////////////////////////// + List newThings = List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ); + + ////////////////////////////// + // replace allowing deletes // + ////////////////////////////// + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("key1", "key2")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newThings); + replaceInput.setFilter(null); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(1, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size()); + assertEquals(1, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTwoKeysWithNullsNotMatchingNotAllowingDelete() throws QException + { + String tableName = TestUtils.TABLE_NAME_TWO_KEYS; + + //////////////////////////////// + // start with these 2 records // + //////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ))); + + //////////////////////////////////////////////////// + // now do a replace action that just updates them // + //////////////////////////////////////////////////// + List newThings = List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ); + + ///////////////////////////////// + // replace disallowing deletes // + ///////////////////////////////// + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("key1", "key2")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newThings); + replaceInput.setFilter(null); + replaceInput.setPerformDeletes(false); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(1, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size()); + assertNull(replaceOutput.getDeleteOutput()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTwoKeysWithNullMatching() throws QException + { + String tableName = TestUtils.TABLE_NAME_TWO_KEYS; + + //////////////////////////////// + // start with these 2 records // + //////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ))); + + //////////////////////////////////////////////////// + // now do a replace action that just updates them // + //////////////////////////////////////////////////// + List newThings = List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ); + + /////////////////////////////////////////////// + // replace treating null key values as equal // + /////////////////////////////////////////////// + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("key1", "key2")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newThings); + replaceInput.setFilter(null); + replaceInput.setAllowNullKeyValuesToEqual(true); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(0, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(2, replaceOutput.getUpdateOutput().getRecords().size()); + assertEquals(0, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -297,4 +450,4 @@ class ReplaceActionTest extends BaseTest return new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(Map.of("firstName", firstName, "lastName", lastName))).getValueInteger("noOfShoes"); } -} \ No newline at end of file +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelperTest.java new file mode 100644 index 00000000..6186ad4e --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelperTest.java @@ -0,0 +1,109 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.tables.helpers; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper.RecordWithErrors; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock.BooleanOperator.AND; + + +/******************************************************************************* + ** Unit test for ValidateRecordSecurityLockHelper + *******************************************************************************/ +class ValidateRecordSecurityLockHelperTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordWithErrors() + { + { + RecordWithErrors recordWithErrors = new RecordWithErrors(new QRecord()); + recordWithErrors.add(new BadInputStatusMessage("0"), List.of(0)); + System.out.println(recordWithErrors); + recordWithErrors.propagateErrorsToRecord(new MultiRecordSecurityLock().withOperator(AND).withLocks(List.of(new RecordSecurityLock()))); + System.out.println("----------------------------------------------------------------------------"); + } + + { + RecordWithErrors recordWithErrors = new RecordWithErrors(new QRecord()); + recordWithErrors.add(new BadInputStatusMessage("1"), List.of(1)); + System.out.println(recordWithErrors); + recordWithErrors.propagateErrorsToRecord(new MultiRecordSecurityLock().withLocks(List.of(new RecordSecurityLock(), new RecordSecurityLock()))); + System.out.println("----------------------------------------------------------------------------"); + } + + { + RecordWithErrors recordWithErrors = new RecordWithErrors(new QRecord()); + recordWithErrors.add(new BadInputStatusMessage("0"), List.of(0)); + recordWithErrors.add(new BadInputStatusMessage("1"), List.of(1)); + System.out.println(recordWithErrors); + recordWithErrors.propagateErrorsToRecord(new MultiRecordSecurityLock().withLocks(List.of(new RecordSecurityLock(), new RecordSecurityLock()))); + System.out.println("----------------------------------------------------------------------------"); + } + + { + RecordWithErrors recordWithErrors = new RecordWithErrors(new QRecord()); + recordWithErrors.add(new BadInputStatusMessage("1,1"), List.of(1, 1)); + System.out.println(recordWithErrors); + recordWithErrors.propagateErrorsToRecord(new MultiRecordSecurityLock().withLocks(List.of( + new MultiRecordSecurityLock().withLocks(List.of(new RecordSecurityLock(), new RecordSecurityLock())), + new MultiRecordSecurityLock().withLocks(List.of(new RecordSecurityLock(), new RecordSecurityLock())) + ))); + System.out.println("----------------------------------------------------------------------------"); + } + + { + RecordWithErrors recordWithErrors = new RecordWithErrors(new QRecord()); + recordWithErrors.add(new BadInputStatusMessage("0,0"), List.of(0, 0)); + recordWithErrors.add(new BadInputStatusMessage("1,1"), List.of(1, 1)); + System.out.println(recordWithErrors); + recordWithErrors.propagateErrorsToRecord(new MultiRecordSecurityLock().withLocks(List.of( + new MultiRecordSecurityLock().withLocks(List.of(new RecordSecurityLock(), new RecordSecurityLock())), + new MultiRecordSecurityLock().withLocks(List.of(new RecordSecurityLock(), new RecordSecurityLock())) + ))); + System.out.println("----------------------------------------------------------------------------"); + } + + { + RecordWithErrors recordWithErrors = new RecordWithErrors(new QRecord()); + recordWithErrors.add(new BadInputStatusMessage("0"), List.of(0)); + recordWithErrors.add(new BadInputStatusMessage("1,1"), List.of(1, 1)); + System.out.println(recordWithErrors); + recordWithErrors.propagateErrorsToRecord(new MultiRecordSecurityLock().withLocks(List.of( + new RecordSecurityLock(), + new MultiRecordSecurityLock().withLocks(List.of(new RecordSecurityLock(), new RecordSecurityLock())) + ))); + System.out.println("----------------------------------------------------------------------------"); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java index 83dd98ef..4249275c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.templates.RenderTemplateInpu import com.kingsrook.qqq.backend.core.model.actions.templates.RenderTemplateOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.templates.TemplateType; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; import org.junit.jupiter.api.Test; @@ -107,8 +108,8 @@ class ConvertHtmlToPdfActionTest extends BaseTest ///////////////////////////////////////////////////////////////////////// // for local dev on a mac, turn this on to auto-open the generated PDF // ///////////////////////////////////////////////////////////////////////// - // todo not commit - // Runtime.getRuntime().exec(new String[] { "/usr/bin/open", "/tmp/file.pdf" }); + // LocalMacDevUtils.mayOpenFiles = true; + LocalMacDevUtils.openFile("/tmp/file.pdf"); } } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java index 893c2a41..8760b0fb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.math.BigDecimal; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -32,10 +33,13 @@ import java.time.ZonedDateTime; import java.util.Collections; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; @@ -210,4 +214,23 @@ class QValueFormatterTest extends BaseTest assertEquals("2023-02-01 07:15:47 PM CST", QValueFormatter.formatDateTimeWithZone(ZonedDateTime.of(LocalDateTime.of(2023, Month.FEBRUARY, 1, 19, 15, 47), ZoneId.of("US/Central")))); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldDisplayBehaviors() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone")); + + QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", "America/Chicago"); + QValueFormatter.setDisplayValuesInRecords(table, List.of(record)); + assertEquals("2024-04-04 02:12:00 PM CDT", record.getDisplayValue("createDate")); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplierTest.java index 96f9d297..65c40b22 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplierTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplierTest.java @@ -29,9 +29,12 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -140,6 +143,36 @@ class ValueBehaviorApplierTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testApplyFormattingBehaviors() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.getField("firstName").withBehavior(ToUpperCaseBehavior.getInstance()); + table.getField("lastName").withBehavior(ToUpperCaseBehavior.NOOP); + table.getField("ssn").withBehavior(ValueTooLongBehavior.TRUNCATE).withMaxLength(1); + + QRecord record = new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("ssn", "0123456789"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + + assertEquals("HOMER", record.getDisplayValue("firstName")); + assertNull(record.getDisplayValue("lastName")); // noop will literally do nothing, not even pass value through. + assertEquals("0123456789", record.getValueString("ssn")); // formatting action should not run the too-long truncate behavior + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now put to-upper-case behavior on lastName, but run INSERT actions - and make sure it doesn't get applied. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + table.getField("lastName").withBehavior(ToUpperCaseBehavior.getInstance()); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null); + assertNull(record.getDisplayValue("lastName")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -153,4 +186,73 @@ class ValueBehaviorApplierTest extends BaseTest return (recordOpt.get()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class ToUpperCaseBehavior implements FieldDisplayBehavior + { + private final boolean enabled; + + private static ToUpperCaseBehavior NOOP = new ToUpperCaseBehavior(false); + private static ToUpperCaseBehavior instance = new ToUpperCaseBehavior(true); + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + private ToUpperCaseBehavior(boolean enabled) + { + this.enabled = enabled; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ToUpperCaseBehavior getDefault() + { + return (NOOP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ToUpperCaseBehavior getInstance() + { + return (instance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + if(!enabled) + { + return; + } + + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + String displayValue = record.getValueString(field.getName()); + if(displayValue != null) + { + displayValue = displayValue.toUpperCase(); + } + + record.setDisplayValue(field.getName(), displayValue); + } + } + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 53e9ec96..20f87686 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; @@ -38,6 +39,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; @@ -233,6 +236,24 @@ class QInstanceEnricherTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInferNameFromBackendName() + { + assertEquals("id", QInstanceEnricher.inferNameFromBackendName("id")); + assertEquals("wordAnotherWordMoreWords", QInstanceEnricher.inferNameFromBackendName("word_another_word_more_words")); + assertEquals("lUlUlUl", QInstanceEnricher.inferNameFromBackendName("l_ul_ul_ul")); + assertEquals("tlaFirst", QInstanceEnricher.inferNameFromBackendName("tla_first")); + assertEquals("wordThenTlaInMiddle", QInstanceEnricher.inferNameFromBackendName("word_then_tla_in_middle")); + assertEquals("endWithTla", QInstanceEnricher.inferNameFromBackendName("end_with_tla")); + assertEquals("tlaAndAnotherTla", QInstanceEnricher.inferNameFromBackendName("tla_and_another_tla")); + assertEquals("allCaps", QInstanceEnricher.inferNameFromBackendName("ALL_CAPS")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -529,4 +550,26 @@ class QInstanceEnricherTest extends BaseTest assertEquals(DynamicDefaultValueBehavior.MODIFY_DATE, table.getField("modifyDate").getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOptionalProcessSteps() + { + QInstance qInstance = TestUtils.defineInstance(); + QProcessMetaData process = new QProcessMetaData(); + process.setName("test"); + process.withStepList(List.of(new QBackendStepMetaData().withName("execute").withCode(new QCodeReference(TestUtils.IncreaseBirthdateStep.class)))); + process.addOptionalStep(new QFrontendStepMetaData() + .withName("screen") + .withViewField(new QFieldMetaData("myField", QFieldType.STRING))); + qInstance.addProcess(process); + + new QInstanceEnricher(qInstance).enrich(); + + assertEquals("My Field", qInstance.getProcess("test").getFrontendStep("screen").getViewFields().get(0).getLabel()); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java index 6e52c49d..3148b508 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java @@ -167,6 +167,36 @@ class QInstanceHelpContentManagerTest extends BaseTest QInstance qInstance = QContext.getQInstance(); new HelpContentMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + HelpContent recordEntity = new HelpContent() + .withId(1) + .withKey("process:" + TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE + ";step:setup") + .withContent("v1") + .withRole(HelpContentRole.PROCESS_SCREEN.getId()); + new InsertAction().execute(new InsertInput(HelpContent.TABLE_NAME).withRecordEntity(recordEntity)); + + /////////////////////////////////////////////////////////////////////////////////////////////// + // now - post-insert customizer should have automatically added help content to the instance // + /////////////////////////////////////////////////////////////////////////////////////////////// + List helpContents = qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getFrontendStep("setup").getHelpContents(); + assertEquals(1, helpContents.size()); + assertEquals("v1", helpContents.get(0).getContent()); + assertEquals(Set.of(QHelpRole.PROCESS_SCREEN), helpContents.get(0).getRoles()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testProcessStep() throws QException + { + ///////////////////////////////////// + // get the instance from base test // + ///////////////////////////////////// + QInstance qInstance = QContext.getQInstance(); + new HelpContentMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + ////////////////////////////////////////////////////////// // first, assert there's no help content on the section // ////////////////////////////////////////////////////////// @@ -216,7 +246,7 @@ class QInstanceHelpContentManagerTest extends BaseTest // now - post-insert customizer should have automatically added help content to the instance // /////////////////////////////////////////////////////////////////////////////////////////////// assertTrue(widget.getHelpContent().containsKey("label")); - assertEquals("i need somebody", widget.getHelpContent().get("label").getContent()); + assertEquals("i need somebody", widget.getHelpContent().get("label").get(0).getContent()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 18786a09..96267363 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -26,8 +26,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -35,6 +38,7 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarCh import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ParentWidgetRenderer; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessActionTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; @@ -52,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -65,14 +70,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; @@ -148,7 +156,7 @@ public class QInstanceValidatorTest extends BaseTest @Test public void test_validateEmptyBackends() { - assertValidationFailureReasons((qInstance) -> qInstance.setBackends(new HashMap<>()), + assertValidationFailureReasonsAllowingExtraReasons((qInstance) -> qInstance.setBackends(new HashMap<>()), "At least 1 backend must be defined"); } @@ -267,6 +275,25 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** Test rules for process step names (must be set; must not be duplicated) + ** + *******************************************************************************/ + @Test + public void test_validateProcessStepNames() + { + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(0).setName(null), + "Missing name for a step at index"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(0).setName(""), + "Missing name for a step at index"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().forEach(s -> s.setName("myStep")), + "Duplicate step name [myStep]", "Duplicate step name [myStep]"); + } + + + /******************************************************************************* ** Test that a process with a step that is a private class fails ** @@ -368,6 +395,147 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test_validateProcessCancelSteps() + { + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).withCancelStep(new QBackendStepMetaData()), + "Cancel step is missing a code reference"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).withCancelStep(new QBackendStepMetaData().withCode(new QCodeReference())), + "missing a code reference name", "missing a code type"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).withCancelStep(new QBackendStepMetaData().withCode(new QCodeReference(ValidAuthCustomizer.class))), + "CodeReference is not of the expected type"); + + assertValidationSuccess((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).withCancelStep(new QBackendStepMetaData().withCode(new QCodeReference(CancelProcessActionTest.CancelStep.class)))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test_validateSchedules() + { + String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE; + Supplier baseScheduleMetaData = () -> new QScheduleMetaData() + .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); + + //////////////////////////////////////////////////// + // do our basic schedule validations on a process // + //////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()), + "either repeatMillis or repeatSeconds or cronExpression must be set"); + + String validCronString = "* * * * * ?"; + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(1) + .withRepeatMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelaySeconds(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelayMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelaySeconds(1) + .withInitialDelayMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression(validCronString)), + "must specify a cronTimeZoneId"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression(validCronString) + .withCronTimeZoneId("foobar")), + "unrecognized cronTimeZoneId: foobar"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression("* * * * * *") + .withCronTimeZoneId("UTC")), + "invalid cron expression: Support for specifying both"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression("x") + .withCronTimeZoneId("UTC")), + "invalid cron expression: Illegal cron expression format"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(10) + .withCronTimeZoneId("UTC")), + "non-cron schedule must not specify a cronTimeZoneId"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withSchedulerName("not-a-scheduler") + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "referencing an unknown scheduler name: not-a-scheduler"); + + ///////////////////////////////// + // validate some success cases // + ///////////////////////////////// + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatSeconds(1))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatMillis(1))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("UTC"))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("America/New_York"))); + + /////////////////////////////////////////////////////////////// + // make sure table automations get their schedules validated // + /////////////////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + //////////////////////////////////////////////////// + // make sure queues get their schedules validated // + //////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> (qInstance.getQueue(TestUtils.TEST_SQS_QUEUE)).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -387,11 +555,12 @@ public class QInstanceValidatorTest extends BaseTest finally { QInstanceValidator.removeAllValidatorPlugins(); - + //////////////////////////////////////////////////// // make sure if remove all plugins, we don't fail // //////////////////////////////////////////////////// - assertValidationSuccess((qInstance) -> {}); + assertValidationSuccess((qInstance) -> { + }); } } @@ -1158,7 +1327,7 @@ public class QInstanceValidatorTest extends BaseTest { TableAutomationAction action = getAction0(qInstance); action.setCodeReference(null); - action.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE); + action.setProcessName(TestUtils.PROCESS_NAME_BASEPULL); }, "different table"); } @@ -1616,6 +1785,26 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldBehaviors() + { + BiFunction fieldExtractor = (QInstance qInstance, String fieldName) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField(fieldName); + assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance, "firstName").withBehaviors(Set.of(ValueTooLongBehavior.ERROR, ValueTooLongBehavior.TRUNCATE)).withMaxLength(1)), + "more than 1 fieldBehavior of type ValueTooLongBehavior, which is not allowed"); + + /////////////////////////////////////////////////////////////////////////// + // make sure a custom validation method in a field behavior gets applied // + // more tests for this particular behavior are in its own test class // + /////////////////////////////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance, "firstName").withBehavior(new DateTimeDisplayValueBehavior())), + "DateTimeDisplayValueBehavior was a applied to a non-DATE_TIME field"); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -1737,6 +1926,7 @@ public class QInstanceValidatorTest extends BaseTest assertValidationFailureReasons((qInstance -> lockExtractor.apply(qInstance).setJoinNameChain(new ArrayList<>())), "looks like a join (has a dot), but no joinNameChain was given"); assertValidationFailureReasons((qInstance -> lockExtractor.apply(qInstance).setFieldName("storeId")), "does not look like a join (does not have a dot), but a joinNameChain was given"); assertValidationFailureReasons((qInstance -> lockExtractor.apply(qInstance).setFieldName("order.wrongId")), "unrecognized fieldName: order.wrongId"); + assertValidationFailureReasons((qInstance -> lockExtractor.apply(qInstance).setFieldName("lineItem.id")), "joinNameChain doesn't end in the expected table [lineItem]"); assertValidationFailureReasons((qInstance -> lockExtractor.apply(qInstance).setJoinNameChain(List.of("notAJoin"))), "an unrecognized join"); assertValidationFailureReasons((qInstance -> lockExtractor.apply(qInstance).setJoinNameChain(List.of("orderLineItem"))), "joinNameChain could not be followed through join"); } @@ -1838,7 +2028,21 @@ public class QInstanceValidatorTest extends BaseTest qInstance.addTable(newTable("B", "id", "aId")); qInstance.addJoin(new QJoinMetaData().withLeftTable("A").withRightTable("B").withName("AB").withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "aId"))); }); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testShareableTableMetaData() + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // just make sure we call this class's validator - the rest of its conditions are covered in its own test // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable("A", "id").withShareableTableMetaData(new ShareableTableMetaData())), + "missing sharedRecordTableName"); } @@ -1947,7 +2151,7 @@ public class QInstanceValidatorTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private QTableMetaData newTable(String tableName, String... fieldNames) + protected QTableMetaData newTable(String tableName, String... fieldNames) { QTableMetaData tableMetaData = new QTableMetaData() .withName(tableName) @@ -2041,7 +2245,7 @@ public class QInstanceValidatorTest extends BaseTest /******************************************************************************* ** Assert that an instance is valid! *******************************************************************************/ - private void assertValidationSuccess(Consumer setup) + public static void assertValidationSuccess(Consumer setup) { try { @@ -2077,7 +2281,7 @@ public class QInstanceValidatorTest extends BaseTest /////////////////////////////////////////////// public abstract class TestAbstractClass extends AbstractTransformStep implements BackendStep { - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { } } @@ -2089,7 +2293,7 @@ public class QInstanceValidatorTest extends BaseTest /////////////////////////////////////////////// private class TestPrivateClass extends AbstractTransformStep implements BackendStep { - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { } @@ -2116,7 +2320,7 @@ public class QInstanceValidatorTest extends BaseTest - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QCollectingLoggerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QCollectingLoggerTest.java new file mode 100644 index 00000000..8a70203e --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QCollectingLoggerTest.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.logging; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.apache.logging.log4j.Level; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for QCollectingLogger + *******************************************************************************/ +class QCollectingLoggerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + ClassThatLogsThings classThatLogsThings = new ClassThatLogsThings(); + classThatLogsThings.logAnInfo("1"); + + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(ClassThatLogsThings.class); + classThatLogsThings.logAnInfo("2"); + classThatLogsThings.logAWarn("3"); + QLogger.deactivateCollectingLoggerForClass(ClassThatLogsThings.class); + + classThatLogsThings.logAWarn("4"); + + assertEquals(2, collectingLogger.getCollectedMessages().size()); + + assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains(""" + "message":"2","""); + assertEquals("2", collectingLogger.getCollectedMessages().get(0).getMessageAsJSONObject().getString("message")); + assertEquals(Level.INFO, collectingLogger.getCollectedMessages().get(0).getLevel()); + + assertThat(collectingLogger.getCollectedMessages().get(1).getMessage()).contains(""" + "message":"3","""); + assertEquals(Level.WARN, collectingLogger.getCollectedMessages().get(1).getLevel()); + assertEquals("3", collectingLogger.getCollectedMessages().get(1).getMessageAsJSONObject().getString("message")); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class ClassThatLogsThings + { + private static final QLogger LOG = QLogger.getLogger(ClassThatLogsThings.class); + + /******************************************************************************* + ** + *******************************************************************************/ + private void logAnInfo(String message) + { + LOG.info(message); + } + + /******************************************************************************* + ** + *******************************************************************************/ + private void logAWarn(String message) + { + LOG.warn(message); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QLoggerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QLoggerTest.java index 1c28982c..de4b60eb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QLoggerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QLoggerTest.java @@ -43,6 +43,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /******************************************************************************* @@ -67,6 +68,50 @@ class QLoggerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLogAndThrowMethods() throws QException + { + try + { + LOG.info("Some info"); + LOG.warnAndThrow(new QException("Something failed"), new LogPair("something", 1)); + } + catch(Exception e) + { + ////////////// + // ok, done // + ////////////// + } + + assertThatThrownBy(() -> + { + try + { + methodThatThrows(); + } + catch(Exception e) + { + throw LOG.errorAndThrow(new QException("I caught, now i errorAndThrow", e), new LogPair("iLove", "logPairs")); + } + } + ).isInstanceOf(QException.class).hasMessageContaining("I caught").rootCause().hasMessageContaining("See, I throw"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void methodThatThrows() throws QException + { + throw (new QException("See, I throw")); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java index 8ef5737d..9f043d1f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java @@ -30,7 +30,7 @@ import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* - ** Unit test for NowWithOffset + ** Unit test for NowWithOffset *******************************************************************************/ class NowWithOffsetTest extends BaseTest { @@ -42,7 +42,7 @@ class NowWithOffsetTest extends BaseTest ** *******************************************************************************/ @Test - void test() + void test() throws Exception { long now = System.currentTimeMillis(); @@ -65,4 +65,4 @@ class NowWithOffsetTest extends BaseTest assertThat(oneYearFromNowMillis).isCloseTo(now + (730 * DAY_IN_MILLIS), Offset.offset(10_000L + 3 * DAY_IN_MILLIS)); } -} \ No newline at end of file +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java index 72f88d3e..0bd1ddf3 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java @@ -31,13 +31,18 @@ import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.json.JSONObject; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.model.data.QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS; import static com.kingsrook.qqq.backend.core.model.data.QRecord.BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; @@ -250,4 +255,40 @@ class QRecordTest extends BaseTest assertNotEquals(originalMap, cloneWithMapValue.getValue("myMap")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetErrorsAndWarningsAsString() + { + assertEquals("", new QRecord().getErrorsAsString()); + assertEquals("one", new QRecord() + .withError(new BadInputStatusMessage("one")) + .getErrorsAsString()); + assertEquals("one; two", new QRecord() + .withError(new BadInputStatusMessage("one")) + .withError(new SystemErrorStatusMessage("two")) + .getErrorsAsString()); + + assertEquals("", new QRecord().getWarningsAsString()); + assertEquals("A", new QRecord() + .withWarning(new QWarningMessage("A")) + .getWarningsAsString()); + assertEquals("A; B; C", new QRecord() + .withWarning(new QWarningMessage("A")) + .withWarning(new QWarningMessage("B")) + .withWarning(new QWarningMessage("C")) + .getWarningsAsString()); + + /////////////////////////////////////////////////////////////////////////////////// + // make sure this AsString method doesn't get included in our json serialization // + /////////////////////////////////////////////////////////////////////////////////// + String json = JsonUtils.toJson(new QRecord() + .withError(new BadInputStatusMessage("one"))); + JSONObject jsonObject = new JSONObject(json); + assertFalse(jsonObject.has("errorsAsString")); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehaviorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehaviorTest.java new file mode 100644 index 00000000..a7afdb22 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehaviorTest.java @@ -0,0 +1,212 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.fields; + + +import java.time.Instant; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for DateTimeDisplayValueBehavior + *******************************************************************************/ +class DateTimeDisplayValueBehaviorTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testZoneIdFromFieldName() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone")); + + QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", "America/Chicago"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertEquals("2024-04-04 02:12:00 PM CDT", record.getDisplayValue("createDate")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testZoneIdFromFieldNameWithFallback() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone").withFallbackZoneId("America/Denver")); + + QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", "whodis"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertEquals("2024-04-04 01:12:00 PM MDT", record.getDisplayValue("createDate")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDefaultZoneId() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withDefaultZoneId("America/Los_Angeles")); + + QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertEquals("2024-04-04 12:12:00 PM PDT", record.getDisplayValue("createDate")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadZoneIdFromOtherField() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone")); + + QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", "fail"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertNull(record.getDisplayValue("createDate")); + + record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", null); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertNull(record.getDisplayValue("createDate")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNullValue() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone")); + + QRecord record = new QRecord().withValue("createDate", null).withValue("timeZone", "UTC"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertNull(record.getDisplayValue("createDate")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidation() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + QFieldMetaData field = table.getField("createDate"); + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + + Function, List> testOne = setup -> + { + DateTimeDisplayValueBehavior dateTimeDisplayValueBehavior = new DateTimeDisplayValueBehavior(); + setup.accept(dateTimeDisplayValueBehavior); + return (dateTimeDisplayValueBehavior.validateBehaviorConfiguration(table, field)); + }; + + /////////////////// + // valid configs // + /////////////////// + assertThat(testOne.apply(b -> b.toString())).isEmpty(); // default setup (noop use-case) is valid + assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone"))).isEmpty(); + assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone").withFallbackZoneId("UTC"))).isEmpty(); + assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone").withFallbackZoneId("America/Chicago"))).isEmpty(); + assertThat(testOne.apply(b -> b.withDefaultZoneId("UTC"))).isEmpty(); + assertThat(testOne.apply(b -> b.withDefaultZoneId("America/Chicago"))).isEmpty(); + + ///////////////////// + // invalid configs // + ///////////////////// + assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("notAField"))) + .hasSize(1).first().asString() + .contains("Unrecognized field name"); + + assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("id"))) + .hasSize(1).first().asString() + .contains("A non-STRING type [INTEGER] was specified as the zoneIdFromFieldName field"); + + assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone").withDefaultZoneId("UTC"))) + .hasSize(1).first().asString() + .contains("You may not specify both zoneIdFromFieldName and defaultZoneId"); + + assertThat(testOne.apply(b -> b.withDefaultZoneId("UTC").withFallbackZoneId("UTC"))) + .hasSize(2) + .anyMatch(s -> s.contains("You may not specify both defaultZoneId and fallbackZoneId")) + .anyMatch(s -> s.contains("You may only set fallbackZoneId if using zoneIdFromFieldName")); + + assertThat(testOne.apply(b -> b.withFallbackZoneId("UTC"))) + .hasSize(1).first().asString() + .contains("You may only set fallbackZoneId if using zoneIdFromFieldName"); + + assertThat(testOne.apply(b -> b.withDefaultZoneId("notAZone"))) + .hasSize(1).first().asString() + .contains("Invalid ZoneId [notAZone] for [defaultZoneId]"); + + assertThat(testOne.apply(b -> b.withZoneIdFromFieldName("timeZone").withFallbackZoneId("notAZone"))) + .hasSize(1).first().asString() + .contains("Invalid ZoneId [notAZone] for [fallbackZoneId]"); + + assertThat(new DateTimeDisplayValueBehavior().validateBehaviorConfiguration(table, table.getField("firstName"))) + .hasSize(1).first().asString() + .contains("non-DATE_TIME field [firstName]"); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java index ad07b882..39d0af14 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -156,4 +157,47 @@ class DynamicDefaultValueBehaviorTest extends BaseTest assertNull(record.getValue("firstName")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUserId() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + table.getField("firstName").withBehavior(DynamicDefaultValueBehavior.USER_ID); + + { + //////////////////////////////// + // set it (if null) on insert // + //////////////////////////////// + QRecord record = new QRecord().withValue("id", 1); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null); + assertEquals(QContext.getQSession().getUser().getIdReference(), record.getValue("firstName")); + } + + { + //////////////////////////////// + // set it (if null) on update // + //////////////////////////////// + QRecord record = new QRecord().withValue("id", 1); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), null); + assertEquals(QContext.getQSession().getUser().getIdReference(), record.getValue("firstName")); + } + + { + //////////////////////////////////////////////////////////////////// + // only set it if it wasn't previously set (both insert & update) // + //////////////////////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 1).withValue("firstName", "Bob"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null); + assertEquals("Bob", record.getValue("firstName")); + + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), null); + assertEquals("Bob", record.getValue("firstName")); + } + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProviderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProviderTest.java new file mode 100644 index 00000000..3f008fd7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/email/EmailMessagingProviderTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging.email; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.messaging.SendMessageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Content; +import com.kingsrook.qqq.backend.core.model.actions.messaging.MultiParty; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Party; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailContentRole; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailPartyRole; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + + +/******************************************************************************* + ** Unit test for EmailMessagingProvider + *******************************************************************************/ +class EmailMessagingProviderTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @DisabledOnOs(OS.LINUX) + void test() throws QException + { + new SendMessageAction().execute(new SendMessageInput() + .withMessagingProviderName(TestUtils.EMAIL_MESSAGING_PROVIDER_NAME) + .withTo(new MultiParty() + .withParty(new Party().withAddress("darin.kelkhoff@gmail.com").withLabel("Darin Kelkhoff").withRole(EmailPartyRole.TO)) + .withParty(new Party().withAddress("james.maes@kingsrook.com").withLabel("Mames Maes").withRole(EmailPartyRole.CC)) + .withParty(new Party().withAddress("tyler.samples@kingsrook.com").withLabel("Tylers Ample").withRole(EmailPartyRole.BCC)) + ) + .withFrom(new Party().withAddress("darin.kelkhoff@gmail.com").withLabel("Darin Kelkhoff")) + .withSubject("This is another qqq test message.") + .withContent(new Content().withContentRole(EmailContentRole.TEXT).withBody("This is a text body")) + .withContent(new Content().withContentRole(EmailContentRole.HTML).withBody("This is an HTML body!")) + ); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESActionTest.java new file mode 100644 index 00000000..16f8792a --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESActionTest.java @@ -0,0 +1,243 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.messaging.ses; + + +import cloud.localstack.ServiceName; +import cloud.localstack.docker.LocalstackDockerExtension; +import cloud.localstack.docker.annotation.LocalstackDockerProperties; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailService; +import com.amazonaws.services.simpleemail.model.Destination; +import com.amazonaws.services.simpleemail.model.Message; +import com.amazonaws.services.simpleemail.model.VerifyEmailAddressRequest; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Content; +import com.kingsrook.qqq.backend.core.model.actions.messaging.MultiParty; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Party; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailContentRole; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailPartyRole; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for SendSESAction + *******************************************************************************/ +@ExtendWith(LocalstackDockerExtension.class) +@LocalstackDockerProperties(useSingleDockerContainer = true, services = { ServiceName.SES }, portEdge = "2960", portElasticSearch = "2961", imageTag = "1.4") +class SendSESActionTest extends BaseTest +{ + public static final String TEST_TO_EMAIL_ADDRESS = "tim-to@coldtrack.com"; + public static final String TEST_FROM_EMAIL_ADDRESS = "tim-from@coldtrack.com"; + + + + /******************************************************************************* + ** Before each unit test, get the test bucket into a known state + *******************************************************************************/ + @BeforeEach + public void beforeEach() + { + AmazonSimpleEmailService amazonSES = getAmazonSES(); + amazonSES.verifyEmailAddress(new VerifyEmailAddressRequest().withEmailAddress(TEST_TO_EMAIL_ADDRESS)); + amazonSES.verifyEmailAddress(new VerifyEmailAddressRequest().withEmailAddress(TEST_FROM_EMAIL_ADDRESS)); + } + + + + /******************************************************************************* + ** Access a localstack-configured SES client. + *******************************************************************************/ + protected AmazonSimpleEmailService getAmazonSES() + { + return (cloud.localstack.awssdkv1.TestUtils.getClientSES()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSendSES() throws QException + { + SendMessageInput sendMessageInput = new SendMessageInput() + .withMessagingProviderName(TestUtils.SES_MESSAGING_PROVIDER_NAME) + .withTo(new MultiParty() + .withParty(new Party().withAddress(TEST_TO_EMAIL_ADDRESS).withLabel("Test TO").withRole(EmailPartyRole.TO)) + ) + .withFrom(new Party().withAddress(TEST_FROM_EMAIL_ADDRESS).withLabel("Test FROM")) + .withSubject("This is another qqq test message.") + .withContent(new Content().withContentRole(EmailContentRole.TEXT).withBody("This is a text body")) + .withContent(new Content().withContentRole(EmailContentRole.HTML).withBody("This is an HTML body!")); + + SendSESAction sendSESAction = new SendSESAction(); + sendSESAction.setAmazonSES(getAmazonSES()); + sendSESAction.sendMessage(sendMessageInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetPartyListFromParty() throws QException + { + SendSESAction sendSESAction = new SendSESAction(); + assertEquals(0, sendSESAction.getPartyListFromParty(null).size()); + assertEquals(1, sendSESAction.getPartyListFromParty(new Party()).size()); + assertEquals(4, sendSESAction.getPartyListFromParty( + new MultiParty() + .withParty(new Party()) + .withParty(new Party()) + .withParty(new Party()) + .withParty(new Party()) + ).size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetSource() throws QException + { + ///////////////////////////////////////////////////////////// + // assert exception if no from given or one without a FROM // + ///////////////////////////////////////////////////////////// + SendSESAction sendSESAction = new SendSESAction(); + assertThatThrownBy(() -> sendSESAction.getSource(new SendMessageInput())).isInstanceOf(QException.class).hasMessageContaining("not provided"); + assertThatThrownBy(() -> sendSESAction.getSource(new SendMessageInput().withFrom(new Party().withRole(EmailPartyRole.REPLY_TO)))).isInstanceOf(QException.class).hasMessageContaining("not provided"); + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // should only be one source, and should be the first one in multi party since multiple not supported // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + SendMessageInput multiPartyInput = new SendMessageInput().withFrom(new MultiParty() + .withParty(new Party().withAddress("test1").withRole(EmailPartyRole.REPLY_TO)) + .withParty(new Party().withAddress("test2").withRole(EmailPartyRole.FROM)) + .withParty(new Party().withAddress("test3").withRole(EmailPartyRole.REPLY_TO)) + .withParty(new Party().withAddress("test4").withRole(EmailPartyRole.FROM)) + ); + assertEquals("test2", sendSESAction.getSource(multiPartyInput)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetReplyTos() throws QException + { + SendMessageInput multiPartyInput = new SendMessageInput() + .withFrom(new MultiParty() + .withParty(new Party().withAddress("test1").withRole(EmailPartyRole.REPLY_TO)) + .withParty(new Party().withAddress("test2").withRole(EmailPartyRole.FROM)) + .withParty(new Party().withAddress("test3").withRole(EmailPartyRole.REPLY_TO)) + .withParty(new Party().withAddress("test4").withRole(EmailPartyRole.FROM)) + ); + SendMessageInput singleInput = new SendMessageInput() + .withFrom(new Party().withAddress("test1").withRole(EmailPartyRole.FROM)); + + SendSESAction sendSESAction = new SendSESAction(); + assertEquals(2, sendSESAction.getReplyTos(multiPartyInput).size()); + assertEquals(0, sendSESAction.getReplyTos(singleInput).size()); + assertEquals(0, sendSESAction.getReplyTos(null).size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBuildDestination() throws QException + { + ///////////////////////////////////////// + // assert exception if no tos provided // + ///////////////////////////////////////// + SendSESAction sendSESAction = new SendSESAction(); + assertThatThrownBy(() -> sendSESAction.buildDestination(new SendMessageInput())).isInstanceOf(QException.class).hasMessageContaining("were provided"); + assertThatThrownBy(() -> sendSESAction.buildDestination(new SendMessageInput().withTo(new Party().withAddress("test1").withRole(EmailPartyRole.CC)))).isInstanceOf(QException.class).hasMessageContaining("were provided"); + + /////////////////////////////////////////// + // exception if a FROM given in to field // + /////////////////////////////////////////// + assertThatThrownBy(() -> sendSESAction.buildDestination(new SendMessageInput().withTo(new Party().withAddress("test1").withRole(EmailPartyRole.FROM)))).isInstanceOf(QException.class).hasMessageContaining("unrecognized"); + + SendMessageInput multiPartyInput = new SendMessageInput() + .withTo(new MultiParty() + .withParty(new Party().withAddress("test1").withRole(EmailPartyRole.CC)) + .withParty(new Party().withAddress("test2").withRole(EmailPartyRole.TO)) + .withParty(new Party().withAddress("test3").withRole(EmailPartyRole.BCC)) + .withParty(new Party().withAddress("test4").withRole(EmailPartyRole.BCC)) + .withParty(new Party().withAddress("test5").withRole(EmailPartyRole.TO)) + .withParty(new Party().withAddress("test6").withRole(EmailPartyRole.TO)) + .withParty(new Party().withAddress("test7").withRole(EmailPartyRole.TO)) + ); + Destination destination = sendSESAction.buildDestination(multiPartyInput); + assertEquals(2, destination.getBccAddresses().size()); + assertEquals(4, destination.getToAddresses().size()); + assertEquals(1, destination.getCcAddresses().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBuildMessage() throws QException + { + ///////////////////////////////////////// + // assert exception if no tos provided // + ///////////////////////////////////////// + SendSESAction sendSESAction = new SendSESAction(); + assertThatThrownBy(() -> sendSESAction.buildMessage(new SendMessageInput())).isInstanceOf(QException.class).hasMessageContaining("'Text' nor an 'HTML'"); + + /////////////////////////////////////////// + // exception if a FROM given in to field // + /////////////////////////////////////////// + assertThatThrownBy(() -> sendSESAction.buildDestination(new SendMessageInput().withTo(new Party().withAddress("test1").withRole(EmailPartyRole.FROM)))).isInstanceOf(QException.class).hasMessageContaining("unrecognized"); + + String htmlContent = "HTML_CONTENT"; + String textContent = "TEXT_CONTENT"; + String subject = "SUBJECT"; + SendMessageInput input = new SendMessageInput() + .withContent(new Content().withContentRole(EmailContentRole.HTML).withBody(htmlContent)) + .withContent(new Content().withContentRole(EmailContentRole.TEXT).withBody(textContent)) + .withSubject(subject); + Message message = sendSESAction.buildMessage(input); + assertEquals(htmlContent, message.getBody().getHtml().getData()); + assertEquals(textContent, message.getBody().getText().getData()); + assertEquals(subject, message.getSubject().getData()); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFiltersTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFiltersTest.java new file mode 100644 index 00000000..b5c360b1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFiltersTest.java @@ -0,0 +1,160 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.security; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock.BooleanOperator.AND; +import static com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock.BooleanOperator.OR; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for RecordSecurityLockFilters + *******************************************************************************/ +class RecordSecurityLockFiltersTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + MultiRecordSecurityLock nullBecauseNull = RecordSecurityLockFilters.filterForReadLockTree(null); + assertNull(nullBecauseNull); + + MultiRecordSecurityLock emptyBecauseEmptyList = RecordSecurityLockFilters.filterForReadLockTree(List.of()); + assertEquals(0, emptyBecauseEmptyList.getLocks().size()); + + MultiRecordSecurityLock emptyBecauseAllWrite = RecordSecurityLockFilters.filterForReadLockTree(List.of( + new RecordSecurityLock().withFieldName("A").withLockScope(RecordSecurityLock.LockScope.WRITE), + new RecordSecurityLock().withFieldName("B").withLockScope(RecordSecurityLock.LockScope.WRITE) + )); + assertEquals(0, emptyBecauseAllWrite.getLocks().size()); + + MultiRecordSecurityLock onlyA = RecordSecurityLockFilters.filterForReadLockTree(List.of( + new RecordSecurityLock().withFieldName("A").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("B").withLockScope(RecordSecurityLock.LockScope.WRITE) + )); + assertMultiRecordSecurityLock(onlyA, AND, "A"); + + MultiRecordSecurityLock twoOutOfThreeTopLevel = RecordSecurityLockFilters.filterForReadLockTree(List.of( + new RecordSecurityLock().withFieldName("A").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("B").withLockScope(RecordSecurityLock.LockScope.WRITE), + new RecordSecurityLock().withFieldName("C").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE) + )); + assertMultiRecordSecurityLock(twoOutOfThreeTopLevel, AND, "A", "C"); + + MultiRecordSecurityLock treeOfAllReads = RecordSecurityLockFilters.filterForReadLockTree(List.of( + new MultiRecordSecurityLock().withOperator(OR).withLocks(List.of( + new RecordSecurityLock().withFieldName("A").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("B").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE) + )), + new MultiRecordSecurityLock().withOperator(OR).withLocks(List.of( + new RecordSecurityLock().withFieldName("C").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("D").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE) + )) + )); + assertEquals(2, treeOfAllReads.getLocks().size()); + assertEquals(AND, treeOfAllReads.getOperator()); + assertMultiRecordSecurityLock((MultiRecordSecurityLock) treeOfAllReads.getLocks().get(0), OR, "A", "B"); + assertMultiRecordSecurityLock((MultiRecordSecurityLock) treeOfAllReads.getLocks().get(1), OR, "C", "D"); + + MultiRecordSecurityLock treeWithOneBranchReadsOneBranchWrites = RecordSecurityLockFilters.filterForReadLockTree(List.of( + new MultiRecordSecurityLock().withOperator(OR).withLocks(List.of( + new RecordSecurityLock().withFieldName("A").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("B").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE) + )), + new MultiRecordSecurityLock().withOperator(OR).withLocks(List.of( + new RecordSecurityLock().withFieldName("C").withLockScope(RecordSecurityLock.LockScope.WRITE), + new RecordSecurityLock().withFieldName("D").withLockScope(RecordSecurityLock.LockScope.WRITE) + )) + )); + assertEquals(2, treeWithOneBranchReadsOneBranchWrites.getLocks().size()); + assertEquals(AND, treeWithOneBranchReadsOneBranchWrites.getOperator()); + assertMultiRecordSecurityLock((MultiRecordSecurityLock) treeWithOneBranchReadsOneBranchWrites.getLocks().get(0), OR, "A", "B"); + assertMultiRecordSecurityLock((MultiRecordSecurityLock) treeWithOneBranchReadsOneBranchWrites.getLocks().get(1), OR); + + MultiRecordSecurityLock deepSparseTree = RecordSecurityLockFilters.filterForReadLockTree(List.of( + new MultiRecordSecurityLock().withOperator(OR).withLocks(List.of( + new MultiRecordSecurityLock().withOperator(AND).withLocks(List.of( + new RecordSecurityLock().withFieldName("A").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("B").withLockScope(RecordSecurityLock.LockScope.WRITE) + )), + new RecordSecurityLock().withFieldName("C").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("D").withLockScope(RecordSecurityLock.LockScope.WRITE) + )), + new MultiRecordSecurityLock().withOperator(OR).withLocks(List.of( + new MultiRecordSecurityLock().withOperator(AND).withLocks(List.of( + new RecordSecurityLock().withFieldName("E").withLockScope(RecordSecurityLock.LockScope.WRITE), + new RecordSecurityLock().withFieldName("F").withLockScope(RecordSecurityLock.LockScope.WRITE) + )), + new MultiRecordSecurityLock().withOperator(AND).withLocks(List.of( + new RecordSecurityLock().withFieldName("G").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("H").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE) + )) + )), + new RecordSecurityLock().withFieldName("I").withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE), + new RecordSecurityLock().withFieldName("J").withLockScope(RecordSecurityLock.LockScope.WRITE) + )); + + assertEquals(3, deepSparseTree.getLocks().size()); + assertEquals(AND, deepSparseTree.getOperator()); + MultiRecordSecurityLock deepChild0 = (MultiRecordSecurityLock) deepSparseTree.getLocks().get(0); + assertEquals(2, deepChild0.getLocks().size()); + assertEquals(OR, deepChild0.getOperator()); + MultiRecordSecurityLock deepGrandChild0 = (MultiRecordSecurityLock) deepChild0.getLocks().get(0); + assertMultiRecordSecurityLock(deepGrandChild0, AND, "A"); + assertEquals("C", deepChild0.getLocks().get(1).getFieldName()); + + MultiRecordSecurityLock deepChild1 = (MultiRecordSecurityLock) deepSparseTree.getLocks().get(1); + assertEquals(2, deepChild1.getLocks().size()); + assertEquals(OR, deepChild1.getOperator()); + MultiRecordSecurityLock deepGrandChild1 = (MultiRecordSecurityLock) deepChild1.getLocks().get(0); + assertMultiRecordSecurityLock(deepGrandChild1, AND); + MultiRecordSecurityLock deepGrandChild2 = (MultiRecordSecurityLock) deepChild1.getLocks().get(1); + assertMultiRecordSecurityLock(deepGrandChild2, AND, "G", "H"); + + assertEquals("I", deepSparseTree.getLocks().get(2).getFieldName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void assertMultiRecordSecurityLock(MultiRecordSecurityLock lock, MultiRecordSecurityLock.BooleanOperator operator, String... lockFieldNames) + { + assertEquals(lockFieldNames.length, lock.getLocks().size()); + assertEquals(operator, lock.getOperator()); + + for(int i = 0; i < lockFieldNames.length; i++) + { + assertEquals(lockFieldNames[i], lock.getLocks().get(i).getFieldName()); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableTableMetaDataTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableTableMetaDataTest.java new file mode 100644 index 00000000..23e5c4a7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/sharing/ShareableTableMetaDataTest.java @@ -0,0 +1,129 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.sharing; + + +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.instances.QInstanceValidatorTest.assertValidationFailureReasons; +import static com.kingsrook.qqq.backend.core.instances.QInstanceValidatorTest.assertValidationFailureReasonsAllowingExtraReasons; +import static com.kingsrook.qqq.backend.core.instances.QInstanceValidatorTest.assertValidationSuccess; + + +/******************************************************************************* + ** Unit test for ShareableTableMetaData + *******************************************************************************/ +class ShareableTableMetaDataTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidation() + { + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData())), + "missing sharedRecordTableName"); + + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName("notATable") + )), "unrecognized sharedRecordTableName"); + + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withAudienceTypesPossibleValueSourceName("notAPVS") + )), "unrecognized audienceTypesPossibleValueSourceName"); + + assertValidationFailureReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + )), "missing assetIdFieldName", + "missing scopeFieldName", + "missing audienceTypes"); + + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withAssetIdFieldName("notAField") + )), "unrecognized assertIdFieldName"); + + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withScopeFieldName("notAField") + )), "unrecognized scopeFieldName"); + + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withAudienceType(new ShareableAudienceType().withName("myType")) + )), "missing fieldName for shareableAudienceType"); + + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withAudienceType(new ShareableAudienceType().withName("myType").withFieldName("notAField")) + )), "unrecognized fieldName"); + + /* todo - corresponding todo in main class + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withAudienceType(new ShareableAudienceType().withName("myType").withFieldName("firstName").withSourceTableName("notATable")) + )), "unrecognized sourceTableName"); + + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withAudienceType(new ShareableAudienceType().withName("myType").withFieldName("firstName").withSourceTableName(TestUtils.TABLE_NAME_SHAPE).withSourceTableKeyFieldName("notAField")) + )), "unrecognized sourceTableKeyFieldName"); + */ + + assertValidationFailureReasonsAllowingExtraReasons(qInstance -> qInstance.addTable(newTable().withShareableTableMetaData(new ShareableTableMetaData() + .withThisTableOwnerIdFieldName("notAField") + )), "unrecognized thisTableOwnerIdFieldName"); + + assertValidationSuccess(qInstance -> qInstance.addTable(newTable() + .withField(new QFieldMetaData("userId", QFieldType.INTEGER)) + .withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withAssetIdFieldName("firstName") + .withScopeFieldName("firstName") + .withThisTableOwnerIdFieldName("userId") + .withAudienceTypesPossibleValueSourceName(TestUtils.POSSIBLE_VALUE_SOURCE_STATE) + .withAudienceType(new ShareableAudienceType().withName("myType").withFieldName("lastName").withSourceTableName(TestUtils.TABLE_NAME_SHAPE).withSourceTableKeyFieldName("id")) + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected QTableMetaData newTable() + { + QTableMetaData tableMetaData = new QTableMetaData() + .withName("A") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id"); + + tableMetaData.addField(new QFieldMetaData("id", QFieldType.INTEGER)); + + return (tableMetaData); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatterTest.java new file mode 100644 index 00000000..4f00be6e --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatterTest.java @@ -0,0 +1,166 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for SavedReportJsonFieldDisplayValueFormatter + *******************************************************************************/ +class SavedReportJsonFieldDisplayValueFormatterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QContext.getQInstance().add(new SavedReportsMetaDataProvider().defineSavedReportTable(TestUtils.MEMORY_BACKEND_NAME, null)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostQuery() throws QException + { + UnsafeFunction customize = savedReport -> + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(SavedReport.TABLE_NAME); + + QRecord record = savedReport.toQRecord(); + + for(String fieldName : List.of("queryFilterJson", "columnsJson", "pivotTableJson")) + { + SavedReportJsonFieldDisplayValueFormatter.getInstance().apply(ValueBehaviorApplier.Action.FORMATTING, List.of(record), qInstance, table, table.getField(fieldName)); + } + + return (record); + }; + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(new ReportColumns())) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition()))); + + assertEquals("0 Filters", record.getDisplayValue("queryFilterJson")); + assertEquals("0 Columns", record.getDisplayValue("columnsJson")); + assertEquals("0 Rows, 0 Columns, and 0 Values", record.getDisplayValue("pivotTableJson")); + } + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(new ReportColumns()))); + + assertEquals("0 Filters", record.getDisplayValue("queryFilterJson")); + assertEquals("0 Columns", record.getDisplayValue("columnsJson")); + assertNull(record.getDisplayValue("pivotTableJson")); + } + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.IS_NOT_BLANK)))) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn(new ReportColumn().withName("birthDate")))) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy()) + .withValue(new PivotTableValue()) + ))); + + assertEquals("1 Filter", record.getDisplayValue("queryFilterJson")); + assertEquals("1 Column", record.getDisplayValue("columnsJson")); + assertEquals("1 Row, 0 Columns, and 1 Value", record.getDisplayValue("pivotTableJson")); + } + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, 1)) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.IS_NOT_BLANK)))) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn(new ReportColumn().withName("__check__").withIsVisible(true)) + .withColumn(new ReportColumn().withName("id")) + .withColumn(new ReportColumn().withName("firstName").withIsVisible(true)) + .withColumn(new ReportColumn().withName("lastName").withIsVisible(false)) + .withColumn(new ReportColumn().withName("birthDate")))) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy()) + .withRow(new PivotTableGroupBy()) + .withColumn(new PivotTableGroupBy()) + .withValue(new PivotTableValue()) + .withValue(new PivotTableValue()) + .withValue(new PivotTableValue()) + ))); + + assertEquals("2 Filters", record.getDisplayValue("queryFilterJson")); + assertEquals("3 Columns", record.getDisplayValue("columnsJson")); + assertEquals("2 Rows, 1 Column, and 3 Values", record.getDisplayValue("pivotTableJson")); + } + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson("blah") + .withColumnsJson("") + .withPivotTableJson("{]")); + + assertEquals("Invalid Filter...", record.getDisplayValue("queryFilterJson")); + assertEquals("Invalid Columns...", record.getDisplayValue("columnsJson")); + assertEquals("Invalid Pivot Table...", record.getDisplayValue("pivotTableJson")); + } + + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizerTest.java new file mode 100644 index 00000000..570a3f4d --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizerTest.java @@ -0,0 +1,255 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.FilterVariableExpression; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for SavedReportTableCustomizer + *******************************************************************************/ +class SavedReportTableCustomizerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QContext.getQInstance().add(new SavedReportsMetaDataProvider().defineSavedReportTable(TestUtils.MEMORY_BACKEND_NAME, null)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreInsertAndPreUpdateAreWired() throws QException + { + SavedReport badRecord = new SavedReport() + .withLabel("My Report") + .withTableName("notATable"); + + ///////////////////////////////////////////////////////////////////// + // assertions to apply both to a failed insert and a failed update // + ///////////////////////////////////////////////////////////////////// + Consumer asserter = record -> assertThat(record.getErrors()) + .hasSizeGreaterThanOrEqualTo(2) + .anyMatch(e -> e.getMessage().contains("Unrecognized table name")) + .anyMatch(e -> e.getMessage().contains("must contain at least 1 column")); + + //////////////////////////////////////////////////////////// + // go through insert action, to ensure wired-up correctly // + //////////////////////////////////////////////////////////// + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(badRecord)); + asserter.accept(insertOutput.getRecords().get(0)); + + //////////////////////////////// + // likewise for update action // + //////////////////////////////// + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(SavedReport.TABLE_NAME).withRecordEntity(badRecord)); + asserter.accept(updateOutput.getRecords().get(0)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPrepareFilterVariable() + { + QQueryFilter qQueryFilter = new QQueryFilter(new QFilterCriteria("id", EQUALS, new FilterVariableExpression())); + + QRecord record = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(qQueryFilter)) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("lastName") + .withColumn("birthDate"))) + .toQRecord(); + + new SavedReportTableCustomizer().preValidateRecord(record); + + assertThat(record.getValueString("queryFilterJson").contains("idEquals")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testParseFails() + { + QRecord record = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson("...") + .withColumnsJson("x") + .withPivotTableJson("[") + .toQRecord(); + + new SavedReportTableCustomizer().preValidateRecord(record); + + assertThat(record.getErrors()) + .hasSize(3) + .anyMatch(e -> e.getMessage().contains("Unable to parse queryFilterJson")) + .anyMatch(e -> e.getMessage().contains("Unable to parse columnsJson")) + .anyMatch(e -> e.getMessage().contains("Unable to parse pivotTableJson")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNoColumns() + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // given a reportColumns object, serialize it to json, put it in a saved report record, and run the pre-validator // + // then assert we got error saying there were no columns. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Consumer asserter = reportColumns -> + { + SavedReport savedReport = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(reportColumns)); + + QRecord record = savedReport.toQRecord(); + new SavedReportTableCustomizer().preValidateRecord(record); + + assertThat(record.getErrors()) + .hasSize(1) + .anyMatch(e -> e.getMessage().contains("must contain at least 1 column")); + }; + + asserter.accept(new ReportColumns()); + asserter.accept(new ReportColumns().withColumns(null)); + asserter.accept(new ReportColumns().withColumns(new ArrayList<>())); + asserter.accept(new ReportColumns().withColumn(new ReportColumn() + .withName("id").withIsVisible(false))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotTables() + { + BiConsumer> asserter = (PivotTableDefinition ptd, List expectedAnyMessageToContain) -> + { + SavedReport savedReport = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("lastName") + .withColumn("birthDate"))) + .withPivotTableJson(JsonUtils.toJson(ptd)); + + QRecord record = savedReport.toQRecord(); + new SavedReportTableCustomizer().preValidateRecord(record); + + assertThat(record.getErrors()).hasSize(expectedAnyMessageToContain.size()); + + for(String expected : expectedAnyMessageToContain) + { + assertThat(record.getErrors()) + .anyMatch(e -> e.getMessage().contains(expected)); + } + }; + + asserter.accept(new PivotTableDefinition(), List.of("must contain at least 1 row")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withRow(new PivotTableGroupBy()), + List.of("Missing field name for at least one pivot table row")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withRow(new PivotTableGroupBy().withFieldName("createDate")), + List.of("row is using field (Create Date) which is not an active column")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withColumn(new PivotTableGroupBy()), + List.of("Missing field name for at least one pivot table column")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withColumn(new PivotTableGroupBy().withFieldName("createDate")), + List.of("column is using field (Create Date) which is not an active column")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withValue(new PivotTableValue().withFunction(PivotTableFunction.SUM)), + List.of("Missing field name for at least one pivot table value")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withValue(new PivotTableValue().withFieldName("createDate").withFunction(PivotTableFunction.SUM)), + List.of("value is using field (Create Date) which is not an active column")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withValue(new PivotTableValue().withFieldName("firstName")), + List.of("Missing function for at least one pivot table value")); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java new file mode 100644 index 00000000..09203b82 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java @@ -0,0 +1,384 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.time.Instant; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import org.assertj.core.api.AbstractStringAssert; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.CronTrigger; +import org.quartz.SchedulerException; +import org.quartz.SimpleTrigger; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ScheduledJobTableCustomizer + *******************************************************************************/ +class ScheduledJobTableCustomizerTest extends BaseTest +{ + private static final String GOOD_CRON = "0 * * * * ?"; + private static final String GOOD_CRON_2 = "* * * * * ?"; + private static final String BAD_CRON = "* * * * * *"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + + new ScheduledJobsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + SchedulerTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreInsertAssertValidationErrors() throws QException + { + UnsafeFunction, QRecord, QException> tryToInsert = consumer -> + { + ScheduledJob scheduledJob = new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true); + consumer.accept(scheduledJob); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(scheduledJob)); + return (insertOutput.getRecords().get(0)); + }; + + ///////////////////////////////////////////////////////// + // lambdas to run a test and assert about no of errors // + ///////////////////////////////////////////////////////// + Function> assertOneErrorExtractingMessage = qRecord -> assertThat(qRecord.getErrors()).hasSize(1).first().extracting("message").asString(); + Consumer assertNoErrors = qRecord -> assertThat(qRecord.getErrors()).hasSize(0); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.setId(null))) + .contains("Either Cron Expression or Repeat Seconds must be given"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(1).withCronExpression(GOOD_CRON).withCronTimeZoneId("UTC"))) + .contains("Cron Expression and Repeat Seconds may not both be given"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(null).withCronExpression(GOOD_CRON))) + .contains("If a Cron Expression is given, then a Cron Time Zone is required"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(null).withCronExpression(BAD_CRON).withCronTimeZoneId("UTC"))) + .contains("Support for specifying both a day-of-week AND a day-of-month parameter is not implemented"); + + /////////////////// + // success cases // + /////////////////// + assertNoErrors.accept(tryToInsert.apply(sj -> sj.withCronExpression(GOOD_CRON).withCronTimeZoneId("UTC"))); + assertNoErrors.accept(tryToInsert.apply(sj -> sj.withRepeatSeconds(1))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostInsertActionSchedulesJob() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of( + new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getErrors())); + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getWarnings())); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostInsertActionIssuesWarnings() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of( + new ScheduledJobParameter().withKey("processName").withValue("notAProcess"))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getErrors())); + assertThat(insertOutput.getRecords().get(0).getWarnings()) + .hasSize(1).first().extracting("message").asString() + .contains("Error scheduling job: Unrecognized processName"); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreUpdateAssertValidationErrors() throws QException + { + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1))); + + UnsafeFunction, QRecord, QException> tryToUpdate = consumer -> + { + QRecord record = new QRecord().withValue("id", 1); + consumer.accept(record); + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecord(record)); + return (updateOutput.getRecords().get(0)); + }; + + ///////////////////////////////////////////////////////// + // lambdas to run a test and assert about no of errors // + ///////////////////////////////////////////////////////// + Function> assertOneErrorExtractingMessage = qRecord -> assertThat(qRecord.getErrors()).hasSize(1).first().extracting("message").asString(); + Consumer assertNoErrors = qRecord -> assertThat(qRecord.getErrors()).hasSize(0); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null))) + .contains("Either Cron Expression or Repeat Seconds must be given"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "UTC"))) + .contains("Cron Expression and Repeat Seconds may not both be given"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON))) + .contains("If a Cron Expression is given, then a Cron Time Zone is required"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", BAD_CRON).withValue("cronTimeZoneId", "UTC"))) + .contains("Support for specifying both a day-of-week AND a day-of-month parameter is not implemented"); + + /////////////////// + // success cases // + /////////////////// + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("modifyDate", Instant.now()))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "UTC"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON_2).withValue("cronTimeZoneId", "UTC"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "America/Chicago"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", 1).withValue("cronExpression", null).withValue("cronTimeZoneId", null))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", 2))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostUpdateActionReSchedulesJob() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(SimpleTrigger.class); + + ////////////////////////////////////// + // now do an update, to re-schedule // + ////////////////////////////////////// + new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withId(1) + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(null) + .withCronExpression(GOOD_CRON) + .withCronTimeZoneId("UTC"))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(CronTrigger.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostUpdateActionIssuesWarnings() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(SimpleTrigger.class); + + ////////////////////////////////////// + // now do an update, to re-schedule // + ////////////////////////////////////// + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withId(1) + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(null) + .withCronExpression(GOOD_CRON) + .withCronTimeZoneId("UTC") + .withJobParameters(List.of(new ScheduledJobParameter().withKey("process").withValue("not"))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(updateOutput.getRecords().get(0).getErrors())); + assertThat(updateOutput.getRecords().get(0).getWarnings()) + .hasSize(1).first().extracting("message").asString() + .contains("Missing scheduledJobParameter with key [processName]"); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostDeleteUnschedules() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + + //////////////////////////////////// + // now do a delete, to unschedule // + //////////////////////////////////// + new DeleteAction().execute(new DeleteInput(ScheduledJob.TABLE_NAME).withPrimaryKeys(List.of(1))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java index 0d0336b9..b1ff66ab 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java @@ -89,7 +89,7 @@ class EnumerationCountActionTest extends BaseTest QInstance instance = QContext.getQInstance(); instance.addBackend(new QBackendMetaData() .withName("enum") - .withBackendType("enum") + .withBackendType(EnumerationBackendModule.class) ); instance.addTable(new QTableMetaData() diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java index 909c9f7c..cf44a7b6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java @@ -167,7 +167,7 @@ class EnumerationQueryActionTest extends BaseTest QInstance instance = QContext.getQInstance(); instance.addBackend(new QBackendMetaData() .withName("enum") - .withBackendType("enum") + .withBackendType(EnumerationBackendModule.class) ); instance.addTable(new QTableMetaData() diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java index fe06c6b1..d9f6df9c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java @@ -224,6 +224,9 @@ class MemoryBackendModuleTest extends BaseTest )); new InsertAction().execute(insertInput); + assertEquals(3, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.TRUE)).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.FALSE)).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2))).size()); assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.IN, List.of(3, 4))).size()); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java index 677fb8d8..afc3ad1f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java @@ -23,11 +23,16 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.utils; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -37,6 +42,182 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class BackendQueryFilterUtilsTest { + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDoesRecordMatch_emptyFilters() + { + assertTrue(BackendQueryFilterUtils.doesRecordMatch(null, new QRecord().withValue("a", 1))); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(new QQueryFilter(), new QRecord().withValue("a", 1))); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(new QQueryFilter().withSubFilters(ListBuilder.of(null)), new QRecord().withValue("a", 1))); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(new QQueryFilter().withSubFilters(List.of(new QQueryFilter())), new QRecord().withValue("a", 1))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDoesRecordMatch_singleAnd() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("a", QCriteriaOperator.EQUALS, 1)); + + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 2))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDoesRecordMatch_singleOr() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("a", QCriteriaOperator.EQUALS, 1)); + + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 2))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Test + void testDoesRecordMatch_multipleAnd() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("a", QCriteriaOperator.EQUALS, 1)) + .withCriteria(new QFilterCriteria("b", QCriteriaOperator.EQUALS, 2)); + + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("b", 2))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 2).withValue("b", 2))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("b", 1))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Test + void testDoesRecordMatch_multipleOr() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("a", QCriteriaOperator.EQUALS, 1)) + .withCriteria(new QFilterCriteria("b", QCriteriaOperator.EQUALS, 2)); + + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("b", 2))); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 2).withValue("b", 2))); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("b", 1))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 3).withValue("b", 4))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Test + void testDoesRecordMatch_subFilterAnd() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withSubFilters(List.of( + new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("a", QCriteriaOperator.EQUALS, 1)), + new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("b", QCriteriaOperator.EQUALS, 2)) + )); + + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("b", 2))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 2).withValue("b", 2))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("b", 1))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Test + void testDoesRecordMatch_subFilterOr() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withSubFilters(List.of( + new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("a", QCriteriaOperator.EQUALS, 1)), + new QQueryFilter() + .withCriteria(new QFilterCriteria("b", QCriteriaOperator.EQUALS, 2)) + )); + + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("b", 2))); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 2).withValue("b", 2))); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("b", 1))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 3).withValue("b", 4))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDoesRecordMatch_criteriaHasTableNameNoFieldsDo() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("t.a", QCriteriaOperator.EQUALS, 1)); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDoesRecordMatch_criteriaHasTableNameSomeFieldsDo() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("t.a", QCriteriaOperator.EQUALS, 1)); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // shouldn't find the "a", because "some" fields in here have a prefix (e.g., 's' was a join table, selected with 't' as the main table, which didn't prefix) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("a", 1).withValue("s.b", 2))); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // but this case (contrasted with above) set the record's tableName to "t", so criteria on "t.a" should find field "a" // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withTableName("t").withValue("a", 1).withValue("s.b", 2))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDoesRecordMatch_criteriaHasTableNameMatchingField() + { + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("t.a", QCriteriaOperator.EQUALS, 1)); + assertTrue(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("t.a", 1))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("t.b", 1))); + assertFalse(BackendQueryFilterUtils.doesRecordMatch(filter, new QRecord().withValue("s.a", 1))); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -184,4 +365,94 @@ class BackendQueryFilterUtilsTest assertFalse("Not Darin".matches(pattern)); assertFalse("David".matches(pattern)); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testApplyBooleanOperator() + { + ///////////////////////////// + // tests for operator: AND // + ///////////////////////////// + { + ///////////////////////////////////////////////////////////////////////////////////// + // old value was true; new value is true. // + // result should be true, and we should not be short-circuited (return value null) // + ///////////////////////////////////////////////////////////////////////////////////// + AtomicBoolean accumulator = new AtomicBoolean(true); + assertNull(BackendQueryFilterUtils.applyBooleanOperator(accumulator, true, QQueryFilter.BooleanOperator.AND)); + assertTrue(accumulator.getPlain()); + } + { + ////////////////////////////////////////////////////////////////////////////////////// + // old value was true; new value is false. // + // result should be false, and we should be short-circuited (return value not-null) // + ////////////////////////////////////////////////////////////////////////////////////// + AtomicBoolean accumulator = new AtomicBoolean(true); + assertEquals(Boolean.FALSE, BackendQueryFilterUtils.applyBooleanOperator(accumulator, false, QQueryFilter.BooleanOperator.AND)); + assertFalse(accumulator.getPlain()); + } + { + ////////////////////////////////////////////////////////////////////////////////////// + // old value was false; new value is true. // + // result should be false, and we should be short-circuited (return value not-null) // + ////////////////////////////////////////////////////////////////////////////////////// + AtomicBoolean accumulator = new AtomicBoolean(false); + assertEquals(Boolean.FALSE, BackendQueryFilterUtils.applyBooleanOperator(accumulator, true, QQueryFilter.BooleanOperator.AND)); + assertFalse(accumulator.getPlain()); + } + { + ////////////////////////////////////////////////////////////////////////////////////// + // old value was false; new value is false. // + // result should be false, and we should be short-circuited (return value not-null) // + ////////////////////////////////////////////////////////////////////////////////////// + AtomicBoolean accumulator = new AtomicBoolean(false); + assertEquals(Boolean.FALSE, BackendQueryFilterUtils.applyBooleanOperator(accumulator, false, QQueryFilter.BooleanOperator.AND)); + assertFalse(accumulator.getPlain()); + } + + //////////////////////////// + // tests for operator: OR // + //////////////////////////// + { + ///////////////////////////////////////////////////////////////////////////////////// + // old value was true; new value is true. // + // result should be true, and we should be short-circuited (return value not-null) // + ///////////////////////////////////////////////////////////////////////////////////// + AtomicBoolean accumulator = new AtomicBoolean(true); + assertEquals(Boolean.TRUE, BackendQueryFilterUtils.applyBooleanOperator(accumulator, true, QQueryFilter.BooleanOperator.OR)); + assertTrue(accumulator.getPlain()); + } + { + ////////////////////////////////////////////////////////////////////////////////////// + // old value was true; new value is false. // + // result should be true, and we should be short-circuited (return value not-null) // + ////////////////////////////////////////////////////////////////////////////////////// + AtomicBoolean accumulator = new AtomicBoolean(true); + assertEquals(Boolean.TRUE, BackendQueryFilterUtils.applyBooleanOperator(accumulator, false, QQueryFilter.BooleanOperator.OR)); + assertTrue(accumulator.getPlain()); + } + { + ////////////////////////////////////////////////////////////////////////////////////// + // old value was false; new value is true. // + // result should be false, and we should be short-circuited (return value not-null) // + ////////////////////////////////////////////////////////////////////////////////////// + AtomicBoolean accumulator = new AtomicBoolean(false); + assertEquals(Boolean.TRUE, BackendQueryFilterUtils.applyBooleanOperator(accumulator, true, QQueryFilter.BooleanOperator.OR)); + assertTrue(accumulator.getPlain()); + } + { + ////////////////////////////////////////////////////////////////////////////////////// + // old value was false; new value is false. // + // result should be false, and we should not be short-circuited (return value null) // + ////////////////////////////////////////////////////////////////////////////////////// + AtomicBoolean accumulator = new AtomicBoolean(false); + assertNull(BackendQueryFilterUtils.applyBooleanOperator(accumulator, false, QQueryFilter.BooleanOperator.OR)); + assertFalse(accumulator.getPlain()); + } + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java index 4779fcfe..e93990a1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java @@ -111,7 +111,7 @@ class BulkInsertTransformStepTest extends BaseTest newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); - bulkInsertTransformStep.run(input, output); + bulkInsertTransformStep.runOnePage(input, output); /////////////////////////////////////////////////////// // assert about the records that passed successfully // @@ -193,7 +193,7 @@ class BulkInsertTransformStepTest extends BaseTest newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); - bulkInsertTransformStep.run(input, output); + bulkInsertTransformStep.runOnePage(input, output); /////////////////////////////////////////////////////// // assert that all records pass. diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStepTest.java index 6881944d..421a3cb6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStepTest.java @@ -76,7 +76,7 @@ class LoadViaInsertOrUpdateStepTest extends BaseTest input.setRecords(inputRecordList); input.addValue(LoadViaInsertOrUpdateStep.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_MEMORY); RunBackendStepOutput output = new RunBackendStepOutput(); - new LoadViaInsertOrUpdateStep().run(input, output); + new LoadViaInsertOrUpdateStep().runOnePage(input, output); List qRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY); assertEquals(2, qRecords.size()); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopTransformStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopTransformStepTest.java index 65bb8530..8142237a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopTransformStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopTransformStepTest.java @@ -56,7 +56,7 @@ class NoopTransformStepTest extends BaseTest RunBackendStepOutput output = new RunBackendStepOutput(); NoopTransformStep noopTransformStep = new NoopTransformStep(); - noopTransformStep.run(input, output); + noopTransformStep.runOnePage(input, output); assertEquals(1, output.getRecords().size()); assertEquals(47, output.getRecords().get(0).getValueInteger("id")); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index 90998ebc..0eeb5429 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -419,7 +419,7 @@ public class StreamedETLWithFrontendProcessTest extends BaseTest ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { for(QRecord qRecord : runBackendStepInput.getRecords()) { @@ -452,7 +452,7 @@ public class StreamedETLWithFrontendProcessTest extends BaseTest ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { for(QRecord qRecord : runBackendStepInput.getRecords()) { @@ -518,7 +518,7 @@ public class StreamedETLWithFrontendProcessTest extends BaseTest ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { for(QRecord qRecord : runBackendStepInput.getRecords()) { @@ -552,7 +552,7 @@ public class StreamedETLWithFrontendProcessTest extends BaseTest ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { for(QRecord qRecord : runBackendStepInput.getRecords()) { @@ -584,7 +584,7 @@ public class StreamedETLWithFrontendProcessTest extends BaseTest ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { /////////////////////////////////// // just pass the records through // diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java index de193fc6..d84e0a97 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java @@ -50,7 +50,7 @@ class BasicRunReportProcessTest extends BaseTest void testRunReport() throws QException { QInstance instance = TestUtils.defineInstance(); - QReportMetaData report = GenerateReportActionTest.definePersonShoesPivotReport(true); + QReportMetaData report = GenerateReportActionTest.definePersonShoesSummaryReport(true); QProcessMetaData runReportProcess = BasicRunReportProcess.defineProcessMetaData(); instance.addReport(report); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportProcessTest.java new file mode 100644 index 00000000..3e6efae7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportProcessTest.java @@ -0,0 +1,406 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryStorageAction; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.commons.io.IOUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for RenderSavedReportExecuteStep + *******************************************************************************/ +class RenderSavedReportProcessTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + GenerateReportActionTest.insertPersonRecords(QContext.getQInstance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @Disabled + void testForDevPrintAPivotDefinitionAsJson() + { + System.out.println(JsonUtils.toPrettyJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy() + .withFieldName("homeStateId")) + .withRow(new PivotTableGroupBy() + .withFieldName("firstName")) + .withValue(new PivotTableValue() + .withFieldName("id") + .withFunction(PivotTableFunction.COUNT)) + .withValue(new PivotTableValue() + .withFieldName("cost") + .withFunction(PivotTableFunction.SUM)) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableOnlyReport() throws Exception + { + String label = "Test Report"; + + ////////////////////////////////////////////////////////////////////////////////////////// + // do columns json as a string, rather than a toJson'ed ReportColumns object, // + // to help verify that we don't choke on un-recognized properties (e.g., as QFMD sends) // + ////////////////////////////////////////////////////////////////////////////////////////// + String columnsJson = """ + {"columns":[ + {"name": "k"}, + {"name": "id"}, + {"name": "firstName", "isVisible": true}, + {"name": "lastName", "pinned": "left"}, + {"name": "createDate", "isVisible": false} + ]}"""; + + QRecord savedReport = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withLabel(label) + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withColumnsJson(columnsJson) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + )).getRecords().get(0); + + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.CSV); + + String downloadFileName = runProcessOutput.getValueString("downloadFileName"); + assertThat(downloadFileName) + .startsWith(label + " - ") + .matches(".*\\d\\d\\d\\d-\\d\\d-\\d\\d-\\d\\d\\d\\d.*") + .endsWith(".csv"); + + InputStream inputStream = getInputStream(runProcessOutput); + List lines = IOUtils.readLines(inputStream, StandardCharsets.UTF_8); + + assertEquals(""" + "Id","First Name","Last Name" + """.trim(), lines.get(0)); + assertEquals(""" + "1","Darin","Jonson" + """.trim(), lines.get(1)); + + writeTmpFileAndOpen(inputStream, ".csv"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static InputStream getInputStream(RunProcessOutput runProcessOutput) throws QException + { + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference)); + return inputStream; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTmpFileAndOpen(InputStream inputStream, String suffix) throws IOException + { + // LocalMacDevUtils.mayOpenFiles = true; + if(LocalMacDevUtils.mayOpenFiles) + { + inputStream.reset(); + + File tmpFile = File.createTempFile(getClass().getName(), suffix, new File("/tmp/")); + FileOutputStream fileOutputStream = new FileOutputStream(tmpFile); + inputStream.transferTo(fileOutputStream); + fileOutputStream.close(); + + LocalMacDevUtils.openFile(tmpFile.getAbsolutePath()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QRecord insertBasicSavedPivotReport(String label) throws QException + { + return new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withLabel(label) + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("lastName") + .withColumn("cost") + .withColumn("birthDate") + .withColumn("homeStateId"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy() + .withFieldName("homeStateId")) + .withRow(new PivotTableGroupBy() + .withFieldName("firstName")) + .withValue(new PivotTableValue() + .withFieldName("id") + .withFunction(PivotTableFunction.COUNT)) + .withValue(new PivotTableValue() + .withFieldName("cost") + .withFunction(PivotTableFunction.SUM)) + .withValue(new PivotTableValue() + .withFieldName("birthDate") + .withFunction(PivotTableFunction.MIN)) + .withValue(new PivotTableValue() + .withFieldName("birthDate") + .withFunction(PivotTableFunction.MAX)) + )) + )).getRecords().get(0); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotXlsx() throws Exception + { + String label = "Test Pivot Report"; + QRecord savedReport = insertBasicSavedPivotReport(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.XLSX); + + InputStream inputStream = getInputStream(runProcessOutput); + writeTmpFileAndOpen(inputStream, ".xlsx"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotJson() throws Exception + { + String label = "Test Pivot Report JSON"; + QRecord savedReport = insertBasicSavedPivotReport(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.JSON); + + InputStream inputStream = getInputStream(runProcessOutput); + String json = StringUtils.join("\n", IOUtils.readLines(inputStream, StandardCharsets.UTF_8)); + printReport(json); + + JSONArray jsonArray = new JSONArray(json); + assertEquals(2, jsonArray.length()); + + JSONObject firstView = jsonArray.getJSONObject(0); + assertEquals(label, firstView.getString("name")); + JSONArray firstViewData = firstView.getJSONArray("data"); + assertEquals(6, firstViewData.length()); + assertThat(firstViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Darin"); + + JSONObject pivotView = jsonArray.getJSONObject(1); + assertEquals("Pivot Table", pivotView.getString("name")); + JSONArray pivotViewData = pivotView.getJSONArray("data"); + assertEquals(4, pivotViewData.length()); + assertThat(pivotViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("homeState", "IL") + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("countOfId", 3) + .hasFieldOrPropertyWithValue("sumOfCost", new BigDecimal("1.50")); + assertThat(pivotViewData.getJSONObject(3).toMap()) + .hasFieldOrPropertyWithValue("homeState", "Totals") + .hasFieldOrPropertyWithValue("countOfId", 6) + .hasFieldOrPropertyWithValue("sumOfCost", new BigDecimal("12.00")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void printReport(String report) + { + // System.out.println(report); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotCSV() throws Exception + { + String label = "Test Pivot Report CSV"; + QRecord savedReport = insertBasicSavedPivotReport(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.CSV); + + InputStream inputStream = getInputStream(runProcessOutput); + List csv = IOUtils.readLines(inputStream, StandardCharsets.UTF_8); + System.out.println(csv); + + assertEquals(""" + "Home State","First Name","Count Of Id","Sum Of Cost","Min Of Birth Date","Max Of Birth Date\"""", csv.get(0)); + + assertEquals(""" + "Totals","","6","12.00","1979-12-30","1980-03-20\"""", csv.get(4)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QRecord insertSavedPivotReportWithAllFunctions(String label) throws QException + { + PivotTableDefinition pivotTableDefinition = new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("firstName")); + + for(PivotTableFunction function : PivotTableFunction.values()) + { + pivotTableDefinition.withValue(new PivotTableValue().withFieldName("cost").withFunction(function)); + } + + return new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withLabel(label) + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("cost"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withPivotTableJson(JsonUtils.toJson(pivotTableDefinition)) + )).getRecords().get(0); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotXlsxAllFunctions() throws Exception + { + String label = "Test Pivot Report"; + QRecord savedReport = insertSavedPivotReportWithAllFunctions(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.XLSX); + + String serverFilePath = runProcessOutput.getValueString("serverFilePath"); + System.out.println(serverFilePath); + + InputStream inputStream = getInputStream(runProcessOutput); + writeTmpFileAndOpen(inputStream, ".xlsx"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotCSVAllFunctions() throws Exception + { + String label = "Test Pivot Report CSV"; + QRecord savedReport = insertSavedPivotReportWithAllFunctions(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.CSV); + + InputStream inputStream = getInputStream(runProcessOutput); + List csv = IOUtils.readLines(inputStream, StandardCharsets.UTF_8); + System.out.println(csv); + + assertEquals(""" + "First Name","Average Of Cost","Count Of Cost","Count_nums Of Cost","Max Of Cost","Min Of Cost","Product Of Cost","Std_dev Of Cost","Std_devp Of Cost","Sum Of Cost","Var Of Cost","Varp Of Cost\"""", csv.get(0)); + + assertEquals(""" + "Totals","2.0","6","6","3.50","0.50","5.359375000000","1.6432","1.5000","12.00","2.7000","2.2500\"""", csv.get(4)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static RunProcessOutput runRenderReportProcess(QRecord savedReport, ReportFormatPossibleValueEnum reportFormat) throws QException + { + RunProcessInput input = new RunProcessInput(); + input.setProcessName(RenderSavedReportMetaDataProducer.NAME); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setCallback(QProcessCallbackFactory.forRecord(savedReport)); + input.addValue("reportFormat", reportFormat.getPossibleValueId()); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + return runProcessOutput; + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/DeleteSharedRecordProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/DeleteSharedRecordProcessTest.java new file mode 100644 index 00000000..38a9e731 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/DeleteSharedRecordProcessTest.java @@ -0,0 +1,132 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.savedreports.SharedSavedReport; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for DeleteSharedRecordProcess + *******************************************************************************/ +class DeleteSharedRecordProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + + new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withLabel("Test") + .withColumnsJson(JsonUtils.toJson(new ReportColumns().withColumn("id"))) + )); + + new InsertAction().execute(new InsertInput(SharedSavedReport.TABLE_NAME).withRecordEntity(new SharedSavedReport() + .withSavedReportId(1) + .withUserId(BaseTest.DEFAULT_USER_ID) + .withScope(ShareScope.READ_WRITE.getPossibleValueId()) + )); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFailCases() throws QException + { + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + DeleteSharedRecordProcess processStep = new DeleteSharedRecordProcess(); + + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: tableName"); + input.addValue("tableName", SavedReport.TABLE_NAME); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: recordId"); + input.addValue("recordId", 1); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: shareId"); + input.addValue("shareId", 3); + + /////////////////////////////////////////////////// + // fail because the requested record isn't found // + /////////////////////////////////////////////////// + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Error deleting shared record: No record was found to delete"); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // now fail because a different user (than the owner, who did the initial delete) is trying to share // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + QContext.setQSession(newSession("not-" + DEFAULT_USER_ID)); + input.addValue("shareId", 1); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("not the owner of this record"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccess() throws QException + { + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + DeleteSharedRecordProcess processStep = new DeleteSharedRecordProcess(); + + input.addValue("tableName", SavedReport.TABLE_NAME); + input.addValue("recordId", 1); + input.addValue("shareId", 1); + + ////////////////////////////////////////// + // assert the shared record got deleted // + ////////////////////////////////////////// + processStep.run(input, output); + + QRecord sharedSavedReportRecord = new GetAction().executeForRecord(new GetInput(SharedSavedReport.TABLE_NAME).withPrimaryKey(1)); + assertNull(sharedSavedReportRecord); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/EditSharedRecordProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/EditSharedRecordProcessTest.java new file mode 100644 index 00000000..1d55f4eb --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/EditSharedRecordProcessTest.java @@ -0,0 +1,144 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.savedreports.SharedSavedReport; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for EditSharedRecordProcess + *******************************************************************************/ +class EditSharedRecordProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + + new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withLabel("Test") + .withColumnsJson(JsonUtils.toJson(new ReportColumns().withColumn("id"))) + )); + + new InsertAction().execute(new InsertInput(SharedSavedReport.TABLE_NAME).withRecordEntity(new SharedSavedReport() + .withSavedReportId(1) + .withUserId(BaseTest.DEFAULT_USER_ID) + .withScope(ShareScope.READ_WRITE.getPossibleValueId()) + )); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFailCases() throws QException + { + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + EditSharedRecordProcess processStep = new EditSharedRecordProcess(); + + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: tableName"); + input.addValue("tableName", SavedReport.TABLE_NAME); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: recordId"); + input.addValue("recordId", 1); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: scopeId"); + input.addValue("scopeId", ShareScope.READ_WRITE.getPossibleValueId()); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: shareId"); + input.addValue("shareId", 3); + + /////////////////////////////////////////////////// + // fail because the requested record isn't found // + /////////////////////////////////////////////////// + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Error editing shared record: No record was found to update"); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // now fail because a different user (than the owner, who did the initial edit) is trying to share // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + QContext.setQSession(newSession("not-" + DEFAULT_USER_ID)); + input.addValue("shareId", 1); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("not the owner of this record"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccess() throws QException + { + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + EditSharedRecordProcess processStep = new EditSharedRecordProcess(); + + /////////////////////////// + // assert original value // + /////////////////////////// + QRecord sharedSavedReportRecord = new GetAction().executeForRecord(new GetInput(SharedSavedReport.TABLE_NAME).withPrimaryKey(1)); + assertEquals(ShareScope.READ_WRITE.getPossibleValueId(), sharedSavedReportRecord.getValueString("scope")); + + input.addValue("tableName", SavedReport.TABLE_NAME); + input.addValue("recordId", 1); + input.addValue("shareId", 1); + input.addValue("scopeId", ShareScope.READ_ONLY.getPossibleValueId()); + + ///////////////////////////////////////// + // assert the shared record got edited // + ///////////////////////////////////////// + processStep.run(input, output); + + ////////////////////////// + // assert updated value // + ////////////////////////// + sharedSavedReportRecord = new GetAction().executeForRecord(new GetInput(SharedSavedReport.TABLE_NAME).withPrimaryKey(1)); + assertEquals(ShareScope.READ_ONLY.getPossibleValueId(), sharedSavedReportRecord.getValueString("scope")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/GetSharedRecordsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/GetSharedRecordsProcessTest.java new file mode 100644 index 00000000..d00e1898 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/GetSharedRecordsProcessTest.java @@ -0,0 +1,98 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.savedreports.SharedSavedReport; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for GetSharedRecordsProcess + *******************************************************************************/ +class GetSharedRecordsProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + + new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withLabel("Test") + .withColumnsJson(JsonUtils.toJson(new ReportColumns().withColumn("id"))))); + + new InsertAction().execute(new InsertInput(SharedSavedReport.TABLE_NAME).withRecordEntity(new SharedSavedReport() + .withScope(ShareScope.READ_WRITE.getPossibleValueId()) + .withUserId("007") + .withSavedReportId(1) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + GetSharedRecordsProcess processStep = new GetSharedRecordsProcess(); + + input.addValue("tableName", SavedReport.TABLE_NAME); + input.addValue("recordId", 1); + processStep.run(input, output); + + List resultList = (List) output.getValue("resultList"); + assertEquals(1, resultList.size()); + + QRecord outputRecord = resultList.get(0); + assertEquals(1, outputRecord.getValueInteger("shareId")); + assertEquals(ShareScope.READ_WRITE.getPossibleValueId(), outputRecord.getValueString("scopeId")); + assertEquals("user", outputRecord.getValueString("audienceType")); + assertEquals("007", outputRecord.getValueString("audienceId")); + assertEquals("user 007", outputRecord.getValueString("audienceLabel")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/InsertSharedRecordProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/InsertSharedRecordProcessTest.java new file mode 100644 index 00000000..67120798 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/InsertSharedRecordProcessTest.java @@ -0,0 +1,146 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.savedreports.SharedSavedReport; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for InsertSharedRecordProcess + *******************************************************************************/ +class InsertSharedRecordProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFailCases() throws QException + { + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + InsertSharedRecordProcess processStep = new InsertSharedRecordProcess(); + + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: tableName"); + input.addValue("tableName", SavedReport.TABLE_NAME); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: recordId"); + input.addValue("recordId", 1); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: audienceType"); + input.addValue("audienceType", "user"); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: audienceId"); + input.addValue("audienceId", "darin@kingsrook.com"); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("Missing required input: scopeId"); + input.addValue("scopeId", ShareScope.READ_WRITE); + + ////////////////////////////// + // try a non-sharable table // + ////////////////////////////// + input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("is not shareable"); + input.addValue("tableName", SavedReport.TABLE_NAME); + + /////////////////////////////////////////////////// + // fail because the requested record isn't found // + /////////////////////////////////////////////////// + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("record could not be found in table, savedReport, with primary key: 1"); + new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withLabel("Test") + .withColumnsJson(JsonUtils.toJson(new ReportColumns().withColumn("id"))) + )); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // now fail because a different user (than the owner, who did the initial insert) is trying to share // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + QContext.setQSession(newSession("not-" + DEFAULT_USER_ID)); + assertThatThrownBy(() -> processStep.run(input, output)).hasMessageContaining("not the owner of this record"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccess() throws QException + { + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + InsertSharedRecordProcess processStep = new InsertSharedRecordProcess(); + + input.addValue("tableName", SavedReport.TABLE_NAME); + input.addValue("recordId", 1); + input.addValue("audienceType", "user"); + input.addValue("audienceId", "darin@kingsrook.com"); + input.addValue("scopeId", ShareScope.READ_WRITE); + + new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withLabel("Test") + .withColumnsJson(JsonUtils.toJson(new ReportColumns().withColumn("id"))) + )); + + //////////////////////////////////////// + // assert the shared record got built // + //////////////////////////////////////// + processStep.run(input, output); + + QRecord sharedSavedReportRecord = new GetAction().executeForRecord(new GetInput(SharedSavedReport.TABLE_NAME).withPrimaryKey(1)); + assertNotNull(sharedSavedReportRecord); + assertEquals(1, sharedSavedReportRecord.getValueInteger("savedReportId")); + assertEquals("darin@kingsrook.com", sharedSavedReportRecord.getValueString("userId")); + assertEquals(ShareScope.READ_WRITE.getPossibleValueId(), sharedSavedReportRecord.getValueString("scope")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharingMetaDataProviderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharingMetaDataProviderTest.java new file mode 100644 index 00000000..841b03dc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/sharing/SharingMetaDataProviderTest.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.sharing; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for SharingMetaDataProvider + *******************************************************************************/ +class SharingMetaDataProviderTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + new SharingMetaDataProvider().defineAll(QContext.getQInstance(), null); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java new file mode 100644 index 00000000..cbe594f6 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java @@ -0,0 +1,383 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.locks; + + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for ProcessLockUtils + *******************************************************************************/ +class ProcessLockUtilsTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QInstance qInstance = QContext.getQInstance(); + MetaDataProducerMultiOutput metaData = new ProcessLockMetaDataProducer().produce(qInstance); + + for(QTableMetaData table : metaData.getEach(QTableMetaData.class)) + { + table.setBackendName(TestUtils.MEMORY_BACKEND_NAME); + } + + metaData.addSelfToInstance(qInstance); + new QInstanceValidator().revalidate(qInstance); + + new InsertAction().execute(new InsertInput(ProcessLockType.TABLE_NAME).withRecordEntities(List.of( + new ProcessLockType() + .withName("typeA") + .withLabel("Type A"), + new ProcessLockType() + .withName("typeB") + .withLabel("Type B") + .withDefaultExpirationSeconds(1), + new ProcessLockType() + .withName("typeC") + .withLabel("Type C") + .withDefaultExpirationSeconds(10) + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ///////////////////////////////////////// + // make sure that we can create a lock // + ///////////////////////////////////////// + ProcessLock processLock = ProcessLockUtils.create("1", "typeA", "me"); + assertNotNull(processLock.getId()); + assertNotNull(processLock.getCheckInTimestamp()); + assertNull(processLock.getExpiresAtTimestamp()); + + ///////////////////////////////////////////////////////// + // make sure we can't create a second for the same key // + ///////////////////////////////////////////////////////// + assertThatThrownBy(() -> ProcessLockUtils.create("1", "typeA", "you")) + .isInstanceOf(UnableToObtainProcessLockException.class) + .hasMessageContaining("Held by: " + QContext.getQSession().getUser().getIdReference()) + .hasMessageContaining("with details: me") + .hasMessageNotContaining("expiring at: 20") + .matches(e -> ((UnableToObtainProcessLockException) e).getExistingLock() != null); + + ///////////////////////////////////////////////////////// + // make sure we can create another for a different key // + ///////////////////////////////////////////////////////// + ProcessLockUtils.create("2", "typeA", "him"); + + ///////////////////////////////////////////////////////////////////// + // make sure we can create another for a different type (same key) // + ///////////////////////////////////////////////////////////////////// + ProcessLockUtils.create("1", "typeB", "her"); + + ////////////////////////////// + // make sure we can release // + ////////////////////////////// + ProcessLockUtils.release(processLock); + + ////////////////////// + // and then you can // + ////////////////////// + processLock = ProcessLockUtils.create("1", "typeA", "you"); + assertNotNull(processLock.getId()); + assertEquals("you", processLock.getDetails()); + + assertThatThrownBy(() -> ProcessLockUtils.create("1", "notAType", "you")) + .isInstanceOf(QException.class) + .hasMessageContaining("Unrecognized process lock type"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSucceedWaitingForExpiration() throws QException + { + ProcessLock processLock = ProcessLockUtils.create("1", "typeB", "me"); + assertNotNull(processLock.getId()); + assertNotNull(processLock.getCheckInTimestamp()); + assertNotNull(processLock.getExpiresAtTimestamp()); + + ///////////////////////////////////////////////////////////////////////// + // make sure someone else can, if they wait longer than the expiration // + ///////////////////////////////////////////////////////////////////////// + processLock = ProcessLockUtils.create("1", "typeB", "you", Duration.of(1, ChronoUnit.SECONDS), Duration.of(3, ChronoUnit.SECONDS)); + assertNotNull(processLock.getId()); + assertThat(processLock.getDetails()).endsWith("you"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFailWaitingForExpiration() throws QException + { + ProcessLock processLock = ProcessLockUtils.create("1", "typeC", "me"); + assertNotNull(processLock.getId()); + assertNotNull(processLock.getCheckInTimestamp()); + assertNotNull(processLock.getExpiresAtTimestamp()); + + ////////////////////////////////////////////////////////////////// + // make sure someone else fails, if they don't wait long enough // + ////////////////////////////////////////////////////////////////// + assertThatThrownBy(() -> ProcessLockUtils.create("1", "typeC", "you", Duration.of(1, ChronoUnit.SECONDS), Duration.of(3, ChronoUnit.SECONDS))) + .isInstanceOf(UnableToObtainProcessLockException.class) + .hasMessageContaining("Held by: " + QContext.getQSession().getUser().getIdReference()) + .hasMessageContaining("with details: me") + .hasMessageContaining("expiring at: 20") + .matches(e -> ((UnableToObtainProcessLockException) e).getExistingLock() != null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCheckInUpdatesExpiration() throws QException + { + ProcessLock processLock = ProcessLockUtils.create("1", "typeB", "me"); + assertNotNull(processLock.getId()); + Instant originalCheckIn = processLock.getCheckInTimestamp(); + Instant originalExpiration = processLock.getExpiresAtTimestamp(); + + SleepUtils.sleep(5, TimeUnit.MILLISECONDS); + ProcessLockUtils.checkIn(processLock); + + ProcessLock freshLock = ProcessLockUtils.getById(processLock.getId()); + assertNotNull(freshLock); + assertNotEquals(originalCheckIn, freshLock.getCheckInTimestamp()); + assertNotEquals(originalExpiration, freshLock.getExpiresAtTimestamp()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testReleaseById() throws QException + { + //////////////////////////////////////////// + // assert no exceptions for these 2 cases // + //////////////////////////////////////////// + ProcessLockUtils.releaseById(null); + ProcessLockUtils.releaseById(1); + + ProcessLock processLock = ProcessLockUtils.create("1", "typeA", "me"); + ProcessLockUtils.releaseById(processLock.getId()); + assertNull(ProcessLockUtils.getById(processLock.getId())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUserAndSessionNullness() throws QException + { + { + QContext.getQSession().setUser(new QUser().withIdReference("me")); + ProcessLock processLock = ProcessLockUtils.create("1", "typeA", null); + assertNull(processLock.getDetails()); + assertEquals("me", processLock.getUserId()); + assertEquals(QContext.getQSession().getUuid(), processLock.getSessionUUID()); + } + + { + ProcessLock processLock = ProcessLockUtils.create("2", "typeA", "foo"); + assertEquals("foo", processLock.getDetails()); + assertEquals("me", processLock.getUserId()); + assertEquals(QContext.getQSession().getUuid(), processLock.getSessionUUID()); + } + + { + QContext.getQSession().setUser(null); + ProcessLock processLock = ProcessLockUtils.create("3", "typeA", "bar"); + assertEquals("bar", processLock.getDetails()); + assertNull(processLock.getUserId()); + assertEquals(QContext.getQSession().getUuid(), processLock.getSessionUUID()); + } + + { + QContext.getQSession().setUuid(null); + ProcessLock processLock = ProcessLockUtils.create("4", "typeA", "baz"); + assertEquals("baz", processLock.getDetails()); + assertNull(processLock.getUserId()); + assertNull(processLock.getSessionUUID()); + } + + { + QContext.getQSession().setUuid(null); + ProcessLock processLock = ProcessLockUtils.create("5", "typeA", ""); + assertEquals("", processLock.getDetails()); + assertNull(processLock.getUserId()); + assertNull(processLock.getSessionUUID()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCheckInExpiresAtTimestampsWithNoDefault() throws QException + { + ///////////////////////////////////////// + // this type has no default expiration // + ///////////////////////////////////////// + ProcessLock processLock = ProcessLockUtils.create("1", "typeA", null); + assertNull(processLock.getExpiresAtTimestamp()); + + ///////////////////////////////////////////////////////////// + // checkin w/o specifying an expires-time - leaves it null // + ///////////////////////////////////////////////////////////// + ProcessLockUtils.checkIn(processLock); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertNull(processLock.getExpiresAtTimestamp()); + + ////////////////////////////////////////////// + // checkin specifying null - leaves it null // + ////////////////////////////////////////////// + ProcessLockUtils.checkIn(processLock, null); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertNull(processLock.getExpiresAtTimestamp()); + + ////////////////////////////////////////////// + // checkin w/ a time - sets it to that time // + ////////////////////////////////////////////// + Instant specifiedTime = Instant.now(); + ProcessLockUtils.checkIn(processLock, specifiedTime); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertEquals(specifiedTime, processLock.getExpiresAtTimestamp()); + + /////////////////////////////////////////////////////////// + // checkin w/o specifying time - leaves it previous time // + /////////////////////////////////////////////////////////// + SleepUtils.sleep(1, TimeUnit.MILLISECONDS); + ProcessLockUtils.checkIn(processLock); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertEquals(specifiedTime, processLock.getExpiresAtTimestamp()); + + //////////////////////////////////////////////////// + // checkin specifying null - puts it back to null // + //////////////////////////////////////////////////// + ProcessLockUtils.checkIn(processLock, null); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertNull(processLock.getExpiresAtTimestamp()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCheckInExpiresAtTimestampsWithSomeDefault() throws QException + { + ///////////////////////////////////////// + // this type has a default expiration // + ///////////////////////////////////////// + ProcessLock processLock = ProcessLockUtils.create("1", "typeB", null); + assertNotNull(processLock.getExpiresAtTimestamp()); + Instant expiresAtTimestamp = processLock.getExpiresAtTimestamp(); + + /////////////////////////////////////////////////////////////// + // checkin w/o specifying an expires-time - moves it forward // + /////////////////////////////////////////////////////////////// + SleepUtils.sleep(1, TimeUnit.MILLISECONDS); + ProcessLockUtils.checkIn(processLock); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertNotNull(processLock.getExpiresAtTimestamp()); + assertNotEquals(expiresAtTimestamp, processLock.getExpiresAtTimestamp()); + + /////////////////////////////////////////////// + // checkin specifying null - sets it to null // + /////////////////////////////////////////////// + ProcessLockUtils.checkIn(processLock, null); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertNull(processLock.getExpiresAtTimestamp()); + + ////////////////////////////////////////////// + // checkin w/ a time - sets it to that time // + ////////////////////////////////////////////// + Instant specifiedTime = Instant.now(); + ProcessLockUtils.checkIn(processLock, specifiedTime); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertEquals(specifiedTime, processLock.getExpiresAtTimestamp()); + + ///////////////////////////////////////////////////////////////////////// + // checkin w/o specifying time - uses the default and moves it forward // + ///////////////////////////////////////////////////////////////////////// + SleepUtils.sleep(1, TimeUnit.MILLISECONDS); + ProcessLockUtils.checkIn(processLock); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertNotEquals(specifiedTime, processLock.getExpiresAtTimestamp()); + + //////////////////////////////////////////////////// + // checkin specifying null - puts it back to null // + //////////////////////////////////////////////////// + ProcessLockUtils.checkIn(processLock, null); + processLock = ProcessLockUtils.getById(processLock.getId()); + assertNull(processLock.getExpiresAtTimestamp()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java new file mode 100644 index 00000000..b33249e1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java @@ -0,0 +1,160 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for QScheduleManager + *******************************************************************************/ +class QScheduleManagerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QLogger.deactivateCollectingLoggerForClass(QuartzScheduler.class); + SchedulerTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSetupScheduledJobErrorCases() throws QException + { + QScheduleManager qScheduleManager = QScheduleManager.initInstance(QContext.getQInstance(), () -> QContext.getQSession()); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withRepeatSeconds(null))) + .hasMessageContaining("Missing a schedule"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withType(null))) + .hasMessageContaining("Missing a type"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withType("notAType"))) + .hasMessageContaining("Unrecognized type"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [processName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", "notAProcess")))) + .hasMessageContaining("Unrecognized processName"); + + QContext.getQInstance().getProcess(TestUtils.PROCESS_NAME_BASEPULL).withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_BASEPULL)))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [queueName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of("queueName", "notAQueue")))) + .hasMessageContaining("Unrecognized queueName"); + + QContext.getQInstance().getQueue(TestUtils.TEST_SQS_QUEUE).withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of("queueName", TestUtils.TEST_SQS_QUEUE)))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [tableName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", "notATable")))) + .hasMessageContaining("Missing scheduledJobParameter with key [automationStatus]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", "notATable", "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("Unrecognized tableName"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("does not have automationDetails"); + + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(null); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", "foobar")))) + .hasMessageContaining("Did not find table automation actions matching automationStatus") + .hasMessageContaining("Found: PENDING_INSERT_AUTOMATIONS,PENDING_UPDATE_AUTOMATIONS"); + + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessfulScheduleWithQuartz() throws QException + { + QCollectingLogger quartzLogger = QLogger.activateCollectingLoggerForClass(QuartzScheduler.class); + + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + + qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, + Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE)) + .withId(2) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + qInstance.getQueue(TestUtils.TEST_SQS_QUEUE).setSchedule(null); + qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, + Map.of("queueName", TestUtils.TEST_SQS_QUEUE)) + .withId(3) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().setSchedule(null); + qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, + Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", AutomationStatus.PENDING_UPDATE_AUTOMATIONS.name())) + .withId(4) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + assertThat(quartzLogger.getCollectedMessages()) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*PROCESS.scheduledJob:2.*")) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*QUEUE_PROCESSOR.scheduledJob:3.*")) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*TABLE_AUTOMATIONS.scheduledJob:4.*")); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java new file mode 100644 index 00000000..439eb47f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java @@ -0,0 +1,140 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.utils.TestUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulerTestUtils +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData buildTestProcess(String name, String schedulerName) + { + return new QProcessMetaData() + .withName(name) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(schedulerName) + .withRepeatMillis(2) + .withInitialDelaySeconds(0)) + .withStepList(List.of(new QBackendStepMetaData() + .withName("step") + .withCode(new QCodeReference(BasicStep.class)))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ScheduledJob newScheduledJob(ScheduledJobType type, Map params) + { + ScheduledJob scheduledJob = new ScheduledJob() + .withId(1) + .withIsActive(true) + .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME) + .withType(type.name()) + .withRepeatSeconds(1) + .withJobParameters(new ArrayList<>()); + + for(Map.Entry entry : params.entrySet()) + { + scheduledJob.getJobParameters().add(new ScheduledJobParameter().withKey(entry.getKey()).withValue(entry.getValue())); + } + + return (scheduledJob); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void afterEach() + { + try + { + QScheduleManager.getInstance().stop(); + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().stop(); + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BasicStep implements BackendStep + { + public static int counter = 0; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + counter++; + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcessTest.java new file mode 100644 index 00000000..e8153867 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcessTest.java @@ -0,0 +1,107 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for RescheduleAllJobsProcess + *******************************************************************************/ +class RescheduleAllJobsProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QLogger.deactivateCollectingLoggerForClass(QuartzScheduler.class); + SchedulerTestUtils.afterEach(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException, SchedulerException + { + QInstance qInstance = QContext.getQInstance(); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, RescheduleAllJobsProcess.class.getPackageName()); + QuartzTestUtils.setupInstanceForQuartzTests(); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + + qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, + Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE)) + .withId(2) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); + List wrappers = quartzScheduler.queryQuartz(); + + /////////////////////////////////////////////////////////////// + // make sure our scheduledJob here got scheduled with quartz // + /////////////////////////////////////////////////////////////// + assertTrue(wrappers.stream().anyMatch(w -> w.jobDetail().getKey().getName().equals("scheduledJob:2"))); + + ///////////////////////// + // run the re-schedule // + ///////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setProcessName(RescheduleAllJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + //////////////////////////////////////////////////////////////////////////////////////// + // now, because our scheduled job record isn't actually stored in ScheduledJob table, // + // when we reschdule all, it should become unscheduled. // + //////////////////////////////////////////////////////////////////////////////////////// + wrappers = quartzScheduler.queryQuartz(); + assertTrue(wrappers.stream().noneMatch(w -> w.jobDetail().getKey().getName().equals("scheduledJob:2"))); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java new file mode 100644 index 00000000..f9db7c70 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobsMetaDataProvider; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ScheduleAllNewJobsProcess + *******************************************************************************/ +class ScheduleAllNewJobsProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + SchedulerTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException, SchedulerException + { + try + { + QCollectingLogger quartzSchedulerLog = QLogger.activateCollectingLoggerForClass(QuartzScheduler.class); + + QInstance qInstance = QContext.getQInstance(); + new ScheduledJobsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, ScheduleAllNewJobsProcess.class.getPackageName()); + QuartzTestUtils.setupInstanceForQuartzTests(); + + /////////////////////////////////////////////////////////////////////////////////// + // clear out the customizers that would normally schedule jobs as we insert them // + /////////////////////////////////////////////////////////////////////////////////// + qInstance.getTable(ScheduledJob.TABLE_NAME).withCustomizers(Collections.emptyMap()); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); + List wrappers = quartzScheduler.queryQuartz(); + + ////////////////////////////////////////////// + // make sure nothing is scheduled initially // + ////////////////////////////////////////////// + assertTrue(wrappers.isEmpty()); + + //////////////////////////////////////////////////////////////////////////// + // insert a scheduled job - run schedule-new, make sure it gets scheduled // + //////////////////////////////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(SchedulerTestUtils + .newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE)) + .withLabel("Test job 1") + .withId(null) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME))); + + RunProcessInput input = new RunProcessInput(); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setProcessName(ScheduleAllNewJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + /////////////////////////////////////////////////////////////// + // make sure our scheduledJob here got scheduled with quartz // + /////////////////////////////////////////////////////////////// + wrappers = quartzScheduler.queryQuartz(); + assertEquals(1, wrappers.size()); + assertTrue(wrappers.stream().anyMatch(w -> w.jobDetail().getKey().getName().equals("scheduledJob:1"))); + + /////////////// + // repeat it // + /////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(SchedulerTestUtils + .newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE)) + .withLabel("Test job 2") + .withId(null) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME))); + + input = new RunProcessInput(); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setProcessName(ScheduleAllNewJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + wrappers = quartzScheduler.queryQuartz(); + assertEquals(2, wrappers.size()); + assertTrue(wrappers.stream().anyMatch(w -> w.jobDetail().getKey().getName().equals("scheduledJob:2"))); + + ///////////////////////////////////////////////////////////////////////////////////// + // make sure quartzScheduler never logged about deleting or re-scheduling anything // + ///////////////////////////////////////////////////////////////////////////////////// + assertThat(quartzSchedulerLog.getCollectedMessages()) + .noneMatch(m -> m.getMessage().toLowerCase().contains("delete")) + .noneMatch(m -> m.getMessage().toLowerCase().contains("re-schedule")); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcessTest.java new file mode 100644 index 00000000..f5774bad --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcessTest.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for UnscheduleAllJobsProcess + *******************************************************************************/ +class UnscheduleAllJobsProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() + { + SchedulerTestUtils.afterEach(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + SchedulerTestUtils.afterEach(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException, SchedulerException + { + QInstance qInstance = QContext.getQInstance(); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, UnscheduleAllJobsProcess.class.getPackageName()); + QuartzTestUtils.setupInstanceForQuartzTests(); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + + qScheduleManager.setupScheduledJob(SchedulerTestUtils.newScheduledJob(ScheduledJobType.PROCESS, + Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE)) + .withId(2) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); + List wrappers = quartzScheduler.queryQuartz(); + assertEquals(1, wrappers.size()); + + RunProcessInput input = new RunProcessInput(); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setProcessName(UnscheduleAllJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + wrappers = quartzScheduler.queryQuartz(); + assertTrue(wrappers.isEmpty()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java new file mode 100644 index 00000000..6d971b1a --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils.BasicStep; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableTableAutomationsRunner; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.Level; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for QuartzScheduler + *******************************************************************************/ +class QuartzSchedulerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + SchedulerTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws Exception + { + try + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // set these runners to use collecting logger, so we can assert that they did run, and didn't throw // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + QCollectingLogger quartzSqsPollerJobLog = QLogger.activateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + QCollectingLogger quartzTableAutomationsJobLog = QLogger.activateCollectingLoggerForClass(SchedulableTableAutomationsRunner.class); + + ////////////////////////////////////////// + // add a process we can run and observe // + ////////////////////////////////////////// + qInstance.addProcess(SchedulerTestUtils.buildTestProcess("testScheduledProcess", QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + ///////////////////////////////////////////////////////////////////// + // start the schedule manager, then ask it to set up all schedules // + ///////////////////////////////////////////////////////////////////// + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + qScheduleManager.setupAllSchedules(); + + ////////////////////////////////////////////////// + // give a moment for the job to run a few times // + ////////////////////////////////////////////////// + SleepUtils.sleep(150, TimeUnit.MILLISECONDS); + qScheduleManager.stopAsync(); + + System.out.println("Ran: " + BasicStep.counter + " times"); + assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s))."); + + ////////////////////////////////////////////////////// + // make sure poller ran, and didn't issue any warns // + ////////////////////////////////////////////////////// + assertThat(quartzSqsPollerJobLog.getCollectedMessages()) + .anyMatch(m -> m.getLevel().equals(Level.DEBUG) && m.getMessage().contains("Running SQS Queue poller")) + .noneMatch(m -> m.getLevel().equals(Level.WARN)); + + ////////////////////////////////////////////////////// + // make sure poller ran, and didn't issue any warns // + ////////////////////////////////////////////////////// + assertThat(quartzTableAutomationsJobLog.getCollectedMessages()) + .anyMatch(m -> m.getLevel().equals(Level.DEBUG) && m.getMessage().contains("Running Table Automations")) + .noneMatch(m -> m.getLevel().equals(Level.WARN)); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + QLogger.deactivateCollectingLoggerForClass(SchedulableTableAutomationsRunner.class); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRemovingNoLongerNeededJobsDuringSetupSchedules() throws SchedulerException + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + //////////////////////////// + // put two jobs in quartz // + //////////////////////////// + QProcessMetaData test1 = SchedulerTestUtils.buildTestProcess("test1", QuartzTestUtils.QUARTZ_SCHEDULER_NAME); + QProcessMetaData test2 = SchedulerTestUtils.buildTestProcess("test2", QuartzTestUtils.QUARTZ_SCHEDULER_NAME); + qInstance.addProcess(test1); + qInstance.addProcess(test2); + + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.name()); + + QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, QuartzTestUtils.QUARTZ_SCHEDULER_NAME, QuartzTestUtils.getQuartzProperties(), () -> QContext.getQSession()); + quartzScheduler.start(); + + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test1", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test2", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); + + quartzScheduler.startOfSetupSchedules(); + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test1", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); + quartzScheduler.endOfSetupSchedules(); + + List quartzJobAndTriggerWrappers = quartzScheduler.queryQuartz(); + assertEquals(1, quartzJobAndTriggerWrappers.size()); + assertEquals("process:test1", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()); + } + + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java new file mode 100644 index 00000000..7e36e6b6 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.List; +import java.util.Properties; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData; +import org.quartz.SchedulerException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzTestUtils +{ + public final static String QUARTZ_SCHEDULER_NAME = "TestQuartzScheduler"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Properties getQuartzProperties() + { + Properties quartzProperties = new Properties(); + quartzProperties.put("org.quartz.scheduler.instanceName", QUARTZ_SCHEDULER_NAME); + quartzProperties.put("org.quartz.threadPool.threadCount", "3"); + quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore"); + return (quartzProperties); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void setupInstanceForQuartzTests() + { + QInstance qInstance = QContext.getQInstance(); + + /////////////////////////////////////////////////// + // remove the simple scheduler from the instance // + /////////////////////////////////////////////////// + qInstance.getSchedulers().clear(); + + //////////////////////////////////////////////////////// + // add the quartz scheduler meta-data to the instance // + //////////////////////////////////////////////////////// + qInstance.addScheduler(new QuartzSchedulerMetaData() + .withProperties(getQuartzProperties()) + .withName(QUARTZ_SCHEDULER_NAME)); + + /////////////////////////////////////////////////////////////////////////////////// + // set the queue providers & automation providers to use the quartz scheduler // + // also, set their initial delay to avoid default delay done by our scheduler // + // (that gives us a chance to re-pause if re-scheduling a previously paused job) // + /////////////////////////////////////////////////////////////////////////////////// + qInstance.getTables().values().forEach(t -> + { + if(t.getAutomationDetails() != null) + { + t.getAutomationDetails().getSchedule() + .withSchedulerName(QUARTZ_SCHEDULER_NAME) + .withInitialDelayMillis(1); + } + }); + + qInstance.getQueues().values() + .forEach(q -> q.getSchedule() + .withSchedulerName(QUARTZ_SCHEDULER_NAME) + .withInitialDelayMillis(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List queryQuartz() throws SchedulerException + { + return QuartzScheduler.getInstance().queryQuartz(); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java new file mode 100644 index 00000000..4f53a495 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -0,0 +1,286 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import org.quartz.Trigger; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit tests for the various quartz management processes + *******************************************************************************/ +class QuartzJobsProcessTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData() + .withName("quartzJobDetails") + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.LONG))); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, QuartzScheduler.class.getPackageName()); + + QuartzTestUtils.setupInstanceForQuartzTests(); + + ////////////////////////////////////////////////////////////////////////////// + // start the schedule manager, which will schedule things, and start quartz // + ////////////////////////////////////////////////////////////////////////////// + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + qScheduleManager.setupAllSchedules(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + SchedulerTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPauseAllQuartzJobs() throws QException, SchedulerException + { + //////////////////////////////////////// + // make sure nothing starts as paused // + //////////////////////////////////////// + assertNoneArePaused(); + + /////////////////////////////// + // run the pause-all process // + /////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ////////////////////////////////////// + // assert everything becomes paused // + ////////////////////////////////////// + assertAllArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testResumeAllQuartzJobs() throws QException, SchedulerException + { + /////////////////////////////// + // run the pause-all process // + /////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + SleepUtils.sleep(3, TimeUnit.SECONDS); + + ////////////////////////////////////// + // assert everything becomes paused // + ////////////////////////////////////// + assertAllArePaused(); + + //////////////////// + // run resume all // + //////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + //////////////////////////////////////// + // make sure nothing ends up as paused // + //////////////////////////////////////// + assertNoneArePaused(); + + //////////////////// + // pause just one // + //////////////////// + List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); + new InsertAction().execute(new InsertInput("quartzJobDetails").withRecord(new QRecord() + .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) + .withValue("jobGroup", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) + )); + + input = new RunProcessInput(); + input.setProcessName(PauseQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ///////////////////////////////////////////////////////// + // make sure at least 1 is paused, some are not paused // + ///////////////////////////////////////////////////////// + assertAnyAre(Trigger.TriggerState.PAUSED); + assertAnyAreNot(Trigger.TriggerState.PAUSED); + + ////////////////////////// + // run resume all again // + ////////////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + //////////////////////////////////////// + // make sure nothing ends up as paused // + //////////////////////////////////////// + assertNoneArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPauseOneResumeOne() throws QException, SchedulerException + { + ///////////////////////////////////// + // make sure nothing starts paused // + ///////////////////////////////////// + assertNoneArePaused(); + + //////////////////// + // pause just one // + //////////////////// + List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); + new InsertAction().execute(new InsertInput("quartzJobDetails").withRecord(new QRecord() + .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) + .withValue("jobGroup", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) + )); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ///////////////////////////////////////////////////////// + // make sure at least 1 is paused, some are not paused // + ///////////////////////////////////////////////////////// + assertAnyAre(Trigger.TriggerState.PAUSED); + assertAnyAreNot(Trigger.TriggerState.PAUSED); + + ///////////////////////////////////////////////////////////////////////////// + // now resume the same one (will still be only row in our in-memory table) // + ///////////////////////////////////////////////////////////////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ////////////////////////////////////// + // make sure nothing ends up paused // + ////////////////////////////////////// + assertNoneArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAnyAre(Trigger.TriggerState triggerState) throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).anyMatch(qjtw -> qjtw.triggerState().equals(triggerState)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAnyAreNot(Trigger.TriggerState triggerState) throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).anyMatch(qjtw -> !qjtw.triggerState().equals(triggerState)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertNoneArePaused() throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).noneMatch(qjtw -> qjtw.triggerState().equals(Trigger.TriggerState.PAUSED)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAllArePaused() throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).allMatch(qjtw -> qjtw.triggerState().equals(Trigger.TriggerState.PAUSED)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java similarity index 55% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index b164376f..d7622e68 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,36 +19,32 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; -import java.util.List; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils.BasicStep; import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** Unit test for ScheduleManager *******************************************************************************/ -class ScheduleManagerTest extends BaseTest +class SimpleSchedulerTest extends BaseTest { /******************************************************************************* @@ -57,7 +53,7 @@ class ScheduleManagerTest extends BaseTest @AfterEach void afterEach() { - ScheduleManager.resetSingleton(); + SchedulerTestUtils.afterEach(); } @@ -66,15 +62,18 @@ class ScheduleManagerTest extends BaseTest ** *******************************************************************************/ @Test - void testStartAndStop() + void testStartAndStop() throws QException { - QInstance qInstance = QContext.getQInstance(); - ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); - scheduleManager.start(); + QInstance qInstance = QContext.getQInstance(); - assertThat(scheduleManager.getExecutors()).isNotEmpty(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); - scheduleManager.stopAsync(); + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + assertThat(simpleScheduler.getExecutors()).isNotEmpty(); + + qScheduleManager.stop(); + simpleScheduler.getExecutors().forEach(e -> assertEquals(StandardScheduledExecutor.RunningState.STOPPED, e.getRunningState())); } @@ -83,50 +82,30 @@ class ScheduleManagerTest extends BaseTest ** *******************************************************************************/ @Test - void testScheduledProcess() throws QInstanceValidationException + void testScheduledProcess() throws QException { QInstance qInstance = QContext.getQInstance(); new QInstanceValidator().validate(qInstance); qInstance.getAutomationProviders().clear(); qInstance.getQueueProviders().clear(); - qInstance.addProcess( - new QProcessMetaData() - .withName("testScheduledProcess") - .withSchedule(new QScheduleMetaData().withRepeatMillis(2).withInitialDelaySeconds(0)) - .withStepList(List.of(new QBackendStepMetaData() - .withName("step") - .withCode(new QCodeReference(BasicStep.class)))) - ); + qInstance.addProcess(SchedulerTestUtils.buildTestProcess("testScheduledProcess", TestUtils.SIMPLE_SCHEDULER_NAME)); BasicStep.counter = 0; - ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); - scheduleManager.setSessionSupplier(QSession::new); - scheduleManager.start(); + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + + ////////////////////////////////////////////////// + // give a moment for the job to run a few times // + ////////////////////////////////////////////////// SleepUtils.sleep(50, TimeUnit.MILLISECONDS); - scheduleManager.stopAsync(); + qScheduleManager.stopAsync(); System.out.println("Ran: " + BasicStep.counter + " times"); - assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice"); + assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s)."); } - - /******************************************************************************* - ** - *******************************************************************************/ - public static class BasicStep implements BackendStep - { - static int counter = 0; - - - - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - counter++; - } - } - } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java index f1dccee9..b8492fe8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java @@ -22,6 +22,8 @@ package com.kingsrook.qqq.backend.core.state; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.UUID; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -88,4 +90,42 @@ public class InMemoryStateProviderTest extends BaseTest }); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testClean() + { + InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance(); + + ///////////////////////////////////////////////////////////// + // Add an entry that is 3 hours old, should not be cleaned // + ///////////////////////////////////////////////////////////// + UUIDAndTypeStateKey newKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS)); + String newUUID = UUID.randomUUID().toString(); + QRecord newQRecord = new QRecord().withValue("uuid", newUUID); + stateProvider.put(newKey, newQRecord); + + //////////////////////////////////////////////////////////// + // Add an entry that is 5 hours old, it should be cleaned // + //////////////////////////////////////////////////////////// + UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS)); + String oldUUID = UUID.randomUUID().toString(); + QRecord oldQRecord = new QRecord().withValue("uuid", oldUUID); + stateProvider.put(oldKey, oldQRecord); + + /////////////////// + // Call to clean // + /////////////////// + stateProvider.clean(Instant.now().minus(4, ChronoUnit.HOURS)); + + QRecord qRecordFromState = stateProvider.get(QRecord.class, newKey).get(); + Assertions.assertEquals(newUUID, qRecordFromState.getValueString("uuid"), "Should read value from state persistence"); + + Assertions.assertTrue(stateProvider.get(QRecord.class, oldKey).isEmpty(), "Key not found in state should return empty"); + + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/testutils/PersonQRecord.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/testutils/PersonQRecord.java index 7f7bdf6f..d5620731 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/testutils/PersonQRecord.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/testutils/PersonQRecord.java @@ -40,6 +40,14 @@ public class PersonQRecord extends QRecord + public PersonQRecord withFirstName(String firstName) + { + setValue("firstName", firstName); + return (this); + } + + + public PersonQRecord withBirthDate(LocalDate birthDate) { setValue("birthDate", birthDate); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ObjectUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ObjectUtilsTest.java index c08486b5..08a308e6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ObjectUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ObjectUtilsTest.java @@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.core.utils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -79,4 +81,20 @@ class ObjectUtilsTest assertEquals("else", ObjectUtils.tryAndRequireNonNullElse(() -> null, "else")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIfCan() + { + Object nullObject = null; + assertTrue(ObjectUtils.ifCan(() -> true)); + assertTrue(ObjectUtils.ifCan(() -> "a".equals("a"))); + assertFalse(ObjectUtils.ifCan(() -> 1 == 2)); + assertFalse(ObjectUtils.ifCan(() -> nullObject.equals("a"))); + assertFalse(ObjectUtils.ifCan(() -> null)); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduperTest.java index 2f596de6..276dcbfb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduperTest.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.GREATER_THAN; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IS_NOT_BLANK; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_EQUALS; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_IN; import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter.BooleanOperator.OR; @@ -352,4 +353,23 @@ class QQueryFilterDeduperTest extends BaseTest assertEquals(contradiction, dedupeFilter(contradiction)); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInAndIsNotBlank() + { + assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", IN, 1, 2)), dedupeFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("f", IN, 1, 2)) + .withCriteria(new QFilterCriteria("f", IS_NOT_BLANK)) + )); + + assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", IN, 1, 2)), dedupeFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("f", IS_NOT_BLANK)) + .withCriteria(new QFilterCriteria("f", IN, 1, 2)) + )); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 2e3a937e..da654f3d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -74,6 +74,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.messaging.email.EmailMessagingProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.messaging.ses.SESMessagingProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; @@ -92,6 +95,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; @@ -135,6 +141,7 @@ public class TestUtils public static final String APP_NAME_PEOPLE = "peopleApp"; public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; + public static final String TABLE_NAME_TWO_KEYS = "twoKeys"; public static final String TABLE_NAME_PERSON = "person"; public static final String TABLE_NAME_SHAPE = "shape"; public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache"; @@ -176,6 +183,12 @@ public class TestUtils public static final String SECURITY_KEY_TYPE_STORE_NULL_BEHAVIOR = "storeNullBehavior"; public static final String SECURITY_KEY_TYPE_INTERNAL_OR_EXTERNAL = "internalOrExternal"; + public static final String EMAIL_MESSAGING_PROVIDER_NAME = "email"; + public static final String SES_MESSAGING_PROVIDER_NAME = "ses"; + + public static final String SIMPLE_SCHEDULER_NAME = "simpleScheduler"; + public static final String TEST_SQS_QUEUE = "testSQSQueue"; + /******************************************************************************* @@ -190,6 +203,7 @@ public class TestUtils qInstance.addBackend(defineMemoryBackend()); qInstance.addTable(defineTablePerson()); + qInstance.addTable(defineTableTwoKeys()); qInstance.addTable(definePersonFileTable()); qInstance.addTable(definePersonMemoryTable()); qInstance.addTable(definePersonMemoryCacheTable()); @@ -234,14 +248,60 @@ public class TestUtils qInstance.addQueueProvider(defineSqsProvider()); qInstance.addQueue(defineTestSqsQueue()); + qInstance.addMessagingProvider(defineEmailMessagingProvider()); + qInstance.addMessagingProvider(defineSESMessagingProvider()); + defineWidgets(qInstance); defineApps(qInstance); + qInstance.addScheduler(defineSimpleScheduler()); + return (qInstance); } + /******************************************************************************* + ** + *******************************************************************************/ + private static QMessagingProviderMetaData defineSESMessagingProvider() + { + String accessKey = "MOCK"; // interpreter.interpret("${env.SES_ACCESS_KEY}"); + String secretKey = "MOCK"; // interpreter.interpret("${env.SES_SECRET_KEY}"); + String region = "MOCK"; // interpreter.interpret("${env.SES_REGION}"); + + return (new SESMessagingProviderMetaData() + .withAccessKey(accessKey) + .withSecretKey(secretKey) + .withRegion(region) + .withName(SES_MESSAGING_PROVIDER_NAME)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QMessagingProviderMetaData defineEmailMessagingProvider() + { + return new EmailMessagingProviderMetaData() + .withSmtpServer("localhost") + .withSmtpPort("2500") + .withName(EMAIL_MESSAGING_PROVIDER_NAME); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QSchedulerMetaData defineSimpleScheduler() + { + return new SimpleSchedulerMetaData().withName(SIMPLE_SCHEDULER_NAME); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -527,6 +587,24 @@ public class TestUtils + /******************************************************************************* + ** Define the 'two key' table used in standard tests. + *******************************************************************************/ + public static QTableMetaData defineTableTwoKeys() + { + return new QTableMetaData() + .withName(TABLE_NAME_TWO_KEYS) + .withLabel("Two Keys") + .withBackendName(MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withUniqueKey(new UniqueKey("key1", "key2")) + .withField(new QFieldMetaData("key1", QFieldType.INTEGER)) + .withField(new QFieldMetaData("key2", QFieldType.INTEGER)); + } + + + /******************************************************************************* ** Define the 'person' table used in standard tests. *******************************************************************************/ @@ -726,6 +804,9 @@ public class TestUtils { return (new QTableAutomationDetails() .withProviderName(POLLING_AUTOMATION) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(SIMPLE_SCHEDULER_NAME) + .withRepeatSeconds(60)) .withStatusTracking(new AutomationStatusTracking() .withType(AutomationStatusTrackingType.FIELD_IN_TABLE) .withFieldName("qqqAutomationStatus"))); @@ -770,6 +851,26 @@ public class TestUtils + /******************************************************************************* + ** Define a table with unique key where one is nullable + *******************************************************************************/ + public static QTableMetaData defineTwoKeyTable() + { + return (new QTableMetaData() + .withName(TABLE_NAME_BASEPULL) + .withLabel("Basepull Test") + .withPrimaryKeyField("id") + .withBackendName(MEMORY_BACKEND_NAME) + .withFields(TestUtils.defineTablePerson().getFields())) + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false)) + .withField(new QFieldMetaData(BASEPULL_KEY_FIELD_NAME, QFieldType.STRING).withBackendName("process_name").withIsRequired(true)) + .withField(new QFieldMetaData(BASEPULL_LAST_RUN_TIME_FIELD_NAME, QFieldType.DATE_TIME).withBackendName("last_run_time").withIsRequired(true)); + } + + + /******************************************************************************* ** Define a basepullTable *******************************************************************************/ @@ -1031,21 +1132,21 @@ public class TestUtils { return new QProcessMetaData() .withName(PROCESS_NAME_GREET_PEOPLE) - .withTableName(TABLE_NAME_PERSON) + .withTableName(TABLE_NAME_PERSON_MEMORY) .addStep(new QBackendStepMetaData() .withName("prepare") .withCode(new QCodeReference() .withName(MockBackendStep.class.getName()) .withCodeType(QCodeType.JAVA)) .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)) + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON_MEMORY)) .withFieldList(List.of( new QFieldMetaData("greetingPrefix", QFieldType.STRING), new QFieldMetaData("greetingSuffix", QFieldType.STRING) ))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName(TABLE_NAME_PERSON) + .withTableName(TABLE_NAME_PERSON_MEMORY) .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) @@ -1324,10 +1425,13 @@ public class TestUtils private static QQueueMetaData defineTestSqsQueue() { return (new QQueueMetaData() - .withName("testSQSQueue") + .withName(TEST_SQS_QUEUE) .withProviderName(DEFAULT_QUEUE_PROVIDER) .withQueueName("test-queue") - .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE)); + .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE) + .withSchedule(new QScheduleMetaData() + .withRepeatSeconds(60) + .withSchedulerName(SIMPLE_SCHEDULER_NAME))); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValidationUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValidationUtilsTest.java new file mode 100644 index 00000000..79ba0351 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValidationUtilsTest.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for ValidationUtils + *******************************************************************************/ +class ValidationUtilsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QUserFacingException + { + assertThatThrownBy(() -> ValidationUtils.parseAndValidateEmailAddresses("notEmail")) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("email addresses were invalid: notEmail"); + + assertThatThrownBy(() -> ValidationUtils.parseAndValidateEmailAddresses("foo@bar.com, whatever")) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("email addresses were invalid: whatever"); + + assertThatThrownBy(() -> ValidationUtils.parseAndValidateEmailAddresses("foo whatever")) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("email addresses were invalid: foo,whatever"); + + assertEquals(List.of("foo@bar.com"), ValidationUtils.parseAndValidateEmailAddresses("foo@bar.com ")); // space here intentional! + assertEquals(List.of("foo@bar.com"), ValidationUtils.parseAndValidateEmailAddresses("foo@bar.com;")); + assertEquals(List.of("foo@bar.com", "fiz@buz.com"), ValidationUtils.parseAndValidateEmailAddresses("foo@bar.com, fiz@buz.com")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java index 86b09257..4dff56c6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.utils.aggregates; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.Month; import com.kingsrook.qqq.backend.core.BaseTest; import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; @@ -78,6 +81,12 @@ class AggregatesTest extends BaseTest assertEquals(15, aggregates.getMax()); assertEquals(30, aggregates.getSum()); assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("750"), aggregates.getProduct()); + assertEquals(new BigDecimal("25.0000"), aggregates.getVariance()); + assertEquals(new BigDecimal("5.0000"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("16.6667"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("4.0824"), Offset.offset(new BigDecimal(".0001"))); } @@ -89,6 +98,7 @@ class AggregatesTest extends BaseTest void testBigDecimal() { BigDecimalAggregates aggregates = new BigDecimalAggregates(); + aggregates.add(null); assertEquals(0, aggregates.getCount()); assertNull(aggregates.getMin()); @@ -114,13 +124,117 @@ class AggregatesTest extends BaseTest BigDecimal bd148 = new BigDecimal("14.8"); aggregates.add(bd148); - - aggregates.add(null); assertEquals(3, aggregates.getCount()); assertEquals(bd51, aggregates.getMin()); assertEquals(bd148, aggregates.getMax()); assertEquals(new BigDecimal("30.0"), aggregates.getSum()); assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10.0"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("762.348"), aggregates.getProduct()); + assertEquals(new BigDecimal("23.5300"), aggregates.getVariance()); + assertEquals(new BigDecimal("4.8508"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("15.6867"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("3.9606"), Offset.offset(new BigDecimal(".0001"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInstant() + { + InstantAggregates aggregates = new InstantAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + Instant i1970 = Instant.parse("1970-01-01T00:00:00Z"); + aggregates.add(i1970); + assertEquals(1, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1970, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(i1970, aggregates.getAverage()); + + Instant i1980 = Instant.parse("1980-01-01T00:00:00Z"); + aggregates.add(i1980); + assertEquals(2, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1980, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(Instant.parse("1975-01-01T00:00:00Z"), aggregates.getAverage()); + + Instant i1990 = Instant.parse("1990-01-01T00:00:00Z"); + aggregates.add(i1990); + assertEquals(3, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1990, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(Instant.parse("1980-01-01T08:00:00Z"), aggregates.getAverage()); // a leap day throws this off by 8 hours :) + + ///////////////////////////////////////////////////////////////////// + // assert we gracefully return null for these ops we don't support // + ///////////////////////////////////////////////////////////////////// + assertNull(aggregates.getProduct()); + assertNull(aggregates.getVariance()); + assertNull(aggregates.getStandardDeviation()); + assertNull(aggregates.getVarP()); + assertNull(aggregates.getStdDevP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLocalDate() + { + LocalDateAggregates aggregates = new LocalDateAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + LocalDate ld1970 = LocalDate.of(1970, Month.JANUARY, 1); + aggregates.add(ld1970); + assertEquals(1, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1970, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(ld1970, aggregates.getAverage()); + + LocalDate ld1980 = LocalDate.of(1980, Month.JANUARY, 1); + aggregates.add(ld1980); + assertEquals(2, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1980, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(LocalDate.of(1975, Month.JANUARY, 1), aggregates.getAverage()); + + LocalDate ld1990 = LocalDate.of(1990, Month.JANUARY, 1); + aggregates.add(ld1990); + assertEquals(3, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1990, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(ld1980, aggregates.getAverage()); + + ///////////////////////////////////////////////////////////////////// + // assert we gracefully return null for these ops we don't support // + ///////////////////////////////////////////////////////////////////// + assertNull(aggregates.getProduct()); + assertNull(aggregates.getVariance()); + assertNull(aggregates.getStandardDeviation()); + assertNull(aggregates.getVarP()); + assertNull(aggregates.getStdDevP()); } } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java index 5d999353..9bc9c367 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java @@ -119,6 +119,57 @@ class MemoizationTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMayNotStoreNull() + { + Memoization memoization = new Memoization<>(); + memoization.setMayStoreNullValues(false); + + AtomicInteger callCounter = new AtomicInteger(); + callCounter.set(0); + UnsafeFunction supplier = name -> + { + callCounter.getAndIncrement(); + if(name.equals("throw")) + { + throw (new Exception("You asked me to throw")); + } + else if(name.equals("null")) + { + return (null); + } + else + { + return (name); + } + }; + + assertThat(memoization.getResult("null", supplier)).isEmpty(); + assertEquals(1, callCounter.get()); + + assertThat(memoization.getResult("null", supplier)).isEmpty(); + assertEquals(2, callCounter.get()); // should re-run the supplier, incrementing the counter + + assertThat(memoization.getResult("throw", supplier)).isEmpty(); + assertEquals(3, callCounter.get()); + + assertThat(memoization.getResult("throw", supplier)).isEmpty(); + assertEquals(4, callCounter.get()); // should re-run the supplier, incrementing the counter + + //noinspection AssertBetweenInconvertibleTypes + assertThat(memoization.getResult("foo", supplier)).isPresent().get().isEqualTo("foo"); + assertEquals(5, callCounter.get()); + + //noinspection AssertBetweenInconvertibleTypes + assertThat(memoization.getResult("foo", supplier)).isPresent().get().isEqualTo("foo"); + assertEquals(5, callCounter.get()); // should NOT re-run the supplier, NOT incrementing the counter + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java index 486d2db6..9214829c 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.api.actions.APICountAction; import com.kingsrook.qqq.backend.module.api.actions.APIDeleteAction; @@ -44,6 +45,11 @@ import com.kingsrook.qqq.backend.module.api.actions.APIUpdateAction; *******************************************************************************/ public class APIBackendModule implements QBackendModuleInterface { + static + { + QBackendModuleDispatcher.registerBackendModule(new APIBackendModule()); + } + /******************************************************************************* ** Method where a backend module must be able to provide its type (name). *******************************************************************************/ diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 5be4635d..57cef42d 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -69,6 +69,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.module.api.exceptions.OAuthCredentialsException; import com.kingsrook.qqq.backend.module.api.exceptions.OAuthExpiredTokenException; +import com.kingsrook.qqq.backend.module.api.exceptions.QBadHttpResponseStatusException; import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException; import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorException; import com.kingsrook.qqq.backend.module.api.model.AuthorizationType; @@ -593,7 +594,7 @@ public class BaseAPIActionUtil } String warningMessage = "HTTP " + request.getMethod() + " for table [" + table.getName() + "] failed with status " + statusCode + ": " + resultString; - throw (new QException(warningMessage)); + throw (new QBadHttpResponseStatusException(warningMessage, response)); } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java new file mode 100644 index 00000000..8587ba29 --- /dev/null +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/QBadHttpResponseStatusException.java @@ -0,0 +1,114 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.api.exceptions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.module.api.actions.QHttpResponse; + + +/******************************************************************************* + ** Exception thrown when an API HTTP request failed due to a bad status code. + ** This exception includes the status code as a field, as well as the full + ** response object. + *******************************************************************************/ +public class QBadHttpResponseStatusException extends QException +{ + private int statusCode; + private QHttpResponse response; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QBadHttpResponseStatusException(String message, QHttpResponse response) + { + super(message); + + this.statusCode = response.getStatusCode(); + this.response = response; + } + + + + /******************************************************************************* + ** Getter for statusCode + *******************************************************************************/ + public int getStatusCode() + { + return (this.statusCode); + } + + + + /******************************************************************************* + ** Setter for statusCode + *******************************************************************************/ + public void setStatusCode(int statusCode) + { + this.statusCode = statusCode; + } + + + + /******************************************************************************* + ** Fluent setter for statusCode + *******************************************************************************/ + public QBadHttpResponseStatusException withStatusCode(int statusCode) + { + this.statusCode = statusCode; + return (this); + } + + + + /******************************************************************************* + ** Getter for response + *******************************************************************************/ + public QHttpResponse getResponse() + { + return (this.response); + } + + + + /******************************************************************************* + ** Setter for response + *******************************************************************************/ + public void setResponse(QHttpResponse response) + { + this.response = response; + } + + + + /******************************************************************************* + ** Fluent setter for response + *******************************************************************************/ + public QBadHttpResponseStatusException withResponse(QHttpResponse response) + { + this.response = response; + return (this); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index df54eef9..c010dad9 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -194,7 +195,6 @@ public abstract class AbstractBaseFilesystemAction /******************************************************************************* ** Generic implementation of the execute method from the QueryInterface *******************************************************************************/ - @SuppressWarnings("checkstyle:Indentation") public QueryOutput executeQuery(QueryInput queryInput) throws QException { preAction(queryInput.getBackend()); @@ -368,13 +368,19 @@ public abstract class AbstractBaseFilesystemAction { try { - Optional tableCustomizer = QCodeLoader.getTableCustomizer(AbstractPostReadFileCustomizer.class, table, FilesystemTableCustomizers.POST_READ_FILE.getRole()); - if(tableCustomizer.isEmpty()) + Optional codeReference = table.getCustomizer(FilesystemTableCustomizers.POST_READ_FILE.getRole()); + if(codeReference.isEmpty()) { return (fileContents); } - return tableCustomizer.get().customizeFileContents(fileContents); + AbstractPostReadFileCustomizer tableCustomizer = QCodeLoader.getAdHoc(AbstractPostReadFileCustomizer.class, codeReference.get()); + if(tableCustomizer == null) + { + return (fileContents); + } + + return tableCustomizer.customizeFileContents(fileContents); } catch(Exception e) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index b98d30f7..19c1601e 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -58,7 +58,6 @@ public class FilesystemTableMetaDataBuilder /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:Indentation") public QTableMetaData buildStandardCardinalityOneTable() { AbstractFilesystemTableBackendDetails tableBackendDetails = switch(backend.getBackendType()) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java index 820915ff..7888080f 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -26,11 +26,13 @@ import java.io.File; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; @@ -39,6 +41,7 @@ import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemCount import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemStorageAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; @@ -53,6 +56,10 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys public static final String BACKEND_TYPE = "filesystem"; + static + { + QBackendModuleDispatcher.registerBackendModule(new FilesystemBackendModule()); + } /******************************************************************************* ** For filesystem backends, get the module-specific action base-class, that helps @@ -152,4 +159,14 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys return (new FilesystemDeleteAction()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return (new FilesystemStorageAction()); + } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index df687e90..c9fb807f 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractF import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import org.apache.commons.io.FileUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -183,7 +184,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction // if the file doesn't exist, just exit with noop. don't throw an error - that should only // // happen if the "contract" of the method is broken, and the file still exists // ////////////////////////////////////////////////////////////////////////////////////////////// - LOG.debug("Not deleting file [{}], because it does not exist.", file); + LOG.debug("Not deleting file, because it does not exist.", logPair("file", file)); return; } @@ -218,7 +219,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ////////////////////////////////////////////////////////////////////////////////////// if(!destinationParent.exists()) { - LOG.debug("Making destination directory {} for move", destinationParent.getAbsolutePath()); + LOG.debug("Making destination directory for move", logPair("directory", destinationParent.getAbsolutePath())); if(!destinationParent.mkdirs()) { throw (new FilesystemException("Failed to make destination directory " + destinationParent.getAbsolutePath() + " to move " + source + " into.")); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageAction.java new file mode 100644 index 00000000..e5a0b4f2 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageAction.java @@ -0,0 +1,114 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.jetbrains.annotations.NotNull; + + +/******************************************************************************* + ** (mass, streamed) storage action for filesystem module + *******************************************************************************/ +public class FilesystemStorageAction extends AbstractFilesystemAction implements QStorageInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + String fullPath = getFullPath(storageInput); + File file = new File(fullPath); + if(!file.getParentFile().exists()) + { + if(!file.getParentFile().mkdirs()) + { + throw (new QException("Could not make directory required for storing file: " + fullPath)); + } + } + + return (new FileOutputStream(fullPath)); + } + catch(IOException e) + { + throw (new QException("IOException creating output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @NotNull + private String getFullPath(StorageInput storageInput) + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + return fullPath; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + return (new FileInputStream(getFullPath(storageInput))); + } + catch(IOException e) + { + throw (new QException("IOException getting input stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDownloadURL(StorageInput storageInput) throws QException + { + return ("file://" + getFullPath(storageInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java index c48d9786..a57f9c1e 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java @@ -101,6 +101,7 @@ public class FilesystemImporterStep implements BackendStep private Function securitySupplier = null; + /******************************************************************************* ** Execute the step - using the request as input, and the result as output. *******************************************************************************/ @@ -373,7 +374,6 @@ public class FilesystemImporterStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:Indentation") List parseFileIntoRecords(RunBackendStepInput runBackendStepInput, String content) throws QException { ///////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index d613dced..8a9a6272 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -25,10 +25,12 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; @@ -36,6 +38,7 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3StorageAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; @@ -48,6 +51,10 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke { public static final String BACKEND_TYPE = "s3"; + static + { + QBackendModuleDispatcher.registerBackendModule(new S3BackendModule()); + } /******************************************************************************* ** For filesystem backends, get the module-specific action base-class, that helps @@ -136,4 +143,15 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke return (new S3DeleteAction()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return new S3StorageAction(); + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index b7b8f999..91383c4b 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -113,7 +113,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction. + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3UploadOutputStream; + + +/******************************************************************************* + ** (mass, streamed) storage action for s3 module + *******************************************************************************/ +public class S3StorageAction extends AbstractS3Action implements QStorageInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + preAction(backend); + + AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + String fullPath = getFullPath(storageInput); + S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(amazonS3, backend.getBucketName(), fullPath); + return (s3UploadOutputStream); + } + catch(Exception e) + { + throw (new QException("Exception creating s3 output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullPath(StorageInput storageInput) + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + + ///////////////////////////////////////////////////////////// + // s3 seems to do better w/o leading slashes, so, strip... // + ///////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = fullPath.substring(1); + } + + return fullPath; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + preAction(backend); + + AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + String fullPath = getFullPath(storageInput); + GetObjectRequest getObjectRequest = new GetObjectRequest(backend.getBucketName(), fullPath); + S3Object s3Object = amazonS3.getObject(getObjectRequest); + S3ObjectInputStream objectContent = s3Object.getObjectContent(); + + return (objectContent); + } + catch(Exception e) + { + throw (new QException("Exception getting s3 input stream for file.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDownloadURL(StorageInput storageInput) throws QException + { + try + { + S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + preAction(backend); + + AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + String fullPath = getFullPath(storageInput); + return (amazonS3.getUrl(backend.getBucketName(), fullPath).toString()); + } + catch(Exception e) + { + throw (new QException("Exception getting the S3 download URL.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void makePublic(StorageInput storageInput) throws QException + { + try + { + S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + preAction(backend); + + AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + String fullPath = getFullPath(storageInput); + amazonS3.setObjectAcl(backend.getBucketName(), fullPath, CannedAccessControlList.PublicRead); + } + catch(Exception e) + { + throw (new QException("Exception making s3 file publicly available.", e)); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java index 652913f7..2475b6b5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -50,19 +50,6 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData - /******************************************************************************* - ** Fluent setter for backendType - ** - *******************************************************************************/ - @Override - public S3BackendMetaData withBackendType(String backendType) - { - setBackendType(backendType); - return this; - } - - - /******************************************************************************* ** Getter for bucketName ** diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java new file mode 100644 index 00000000..f67493ee --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java @@ -0,0 +1,205 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** OutputStream implementation that knows how to stream data into a new S3 file. + ** + ** This will be done using a multipart-upload if the contents are > 5MB - else + ** just a 1-time-call to PutObject + *******************************************************************************/ +public class S3UploadOutputStream extends OutputStream +{ + private static final QLogger LOG = QLogger.getLogger(S3UploadOutputStream.class); + + private final AmazonS3 amazonS3; + private final String bucketName; + private final String key; + + private byte[] buffer = new byte[5 * 1024 * 1024]; + private int offset = 0; + + private InitiateMultipartUploadResult initiateMultipartUploadResult = null; + private List uploadPartResultList = null; + + private boolean isClosed = false; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public S3UploadOutputStream(AmazonS3 amazonS3, String bucketName, String key) + { + this.amazonS3 = amazonS3; + this.bucketName = bucketName; + this.key = key; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void write(int b) throws IOException + { + buffer[offset] = (byte) b; + offset++; + + uploadIfNeeded(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void uploadIfNeeded() + { + if(offset == buffer.length) + { + ////////////////////////////////////////// + // start or continue a multipart upload // + ////////////////////////////////////////// + if(initiateMultipartUploadResult == null) + { + LOG.info("Initiating a multipart upload", logPair("key", key)); + initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key)); + uploadPartResultList = new ArrayList<>(); + } + + LOG.info("Uploading a part", logPair("key", key), logPair("partNumber", uploadPartResultList.size() + 1)); + UploadPartRequest uploadPartRequest = new UploadPartRequest() + .withUploadId(initiateMultipartUploadResult.getUploadId()) + .withPartNumber(uploadPartResultList.size() + 1) + .withInputStream(new ByteArrayInputStream(buffer)) + .withBucketName(bucketName) + .withKey(key) + .withPartSize(buffer.length); + + uploadPartResultList.add(amazonS3.uploadPart(uploadPartRequest)); + + ////////////////// + // reset buffer // + ////////////////// + offset = 0; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void write(byte[] b, int off, int len) throws IOException + { + int bytesToWrite = len; + + while(bytesToWrite > buffer.length - offset) + { + int size = buffer.length - offset; + System.arraycopy(b, off, buffer, offset, size); + offset = buffer.length; + uploadIfNeeded(); + off += size; + bytesToWrite -= size; + } + + int size = len - off; + System.arraycopy(b, off, buffer, offset, size); + offset += size; + uploadIfNeeded(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void close() throws IOException + { + if(isClosed) + { + LOG.debug("Redundant call to close an already-closed S3UploadOutputStream. Returning with noop.", logPair("key", key)); + return; + } + + if(initiateMultipartUploadResult != null) + { + if(offset > 0) + { + ////////////////////////////////////////////////// + // if there's a final part to upload, do it now // + ////////////////////////////////////////////////// + LOG.info("Uploading a part", logPair("key", key), logPair("isFinalPart", true), logPair("partNumber", uploadPartResultList.size() + 1)); + UploadPartRequest uploadPartRequest = new UploadPartRequest() + .withUploadId(initiateMultipartUploadResult.getUploadId()) + .withPartNumber(uploadPartResultList.size() + 1) + .withInputStream(new ByteArrayInputStream(buffer, 0, offset)) + .withBucketName(bucketName) + .withKey(key) + .withPartSize(offset); + uploadPartResultList.add(amazonS3.uploadPart(uploadPartRequest)); + } + + CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest() + .withUploadId(initiateMultipartUploadResult.getUploadId()) + .withPartETags(uploadPartResultList) + .withBucketName(bucketName) + .withKey(key); + CompleteMultipartUploadResult completeMultipartUploadResult = amazonS3.completeMultipartUpload(completeMultipartUploadRequest); + } + else + { + LOG.info("Putting object (non-multipart)", logPair("key", key), logPair("length", offset)); + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(offset); + PutObjectResult putObjectResult = amazonS3.putObject(bucketName, key, new ByteArrayInputStream(buffer, 0, offset), objectMetadata); + } + + isClosed = true; + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index 68c99d28..acad9100 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; @@ -63,16 +64,16 @@ public class TestUtils public static final String BACKEND_NAME_S3 = "s3"; public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; public static final String BACKEND_NAME_MOCK = "mock"; - public static final String BACKEND_NAME_MEMORY = "memory"; + public static final String BACKEND_NAME_MEMORY = "memory"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; - public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; + public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_PERSON_S3 = "person-s3"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; - public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; + public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; @@ -403,7 +404,7 @@ public class TestUtils public static QBackendMetaData defineMockBackend() { return (new QBackendMetaData() - .withBackendType("mock") + .withBackendType(MockBackendModule.class) .withName(BACKEND_NAME_MOCK)); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageActionTest.java new file mode 100644 index 00000000..be47210b --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageActionTest.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +class FilesystemStorageActionTest extends FilesystemActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws Exception + { + String data = "Hellooo, Storage."; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withReference("test.txt"); + + OutputStream outputStream = new StorageAction().createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = new StorageAction().getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageActionTest.java new file mode 100644 index 00000000..4df407ee --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageActionTest.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +public class S3StorageActionTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + String data = "Hellooo, Storage."; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_BLOB_S3).withReference("test.txt"); + + ///////////////////////////////////////////////////////////////////////// + // work directly w/ s3 action class here, so we can set s3 utils in it // + ///////////////////////////////////////////////////////////////////////// + S3StorageAction s3StorageAction = new S3StorageAction(); + s3StorageAction.setS3Utils(getS3Utils()); + OutputStream outputStream = s3StorageAction.createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = s3StorageAction.getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStreamTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStreamTest.java new file mode 100644 index 00000000..96d43bce --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStreamTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for S3UploadOutputStream + *******************************************************************************/ +class S3UploadOutputStreamTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws IOException + { + String bucketName = BaseS3Test.BUCKET_NAME; + String key = "uploader-tests/" + Instant.now().toString() + ".txt"; + + // S3UploadOutputStream outputStream = new S3UploadOutputStream(amazonS3, bucketName, key); + // FileOutputStream outputStream = new FileOutputStream("/tmp/file.json"); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + outputStream.write("[\n1".getBytes(StandardCharsets.UTF_8)); + for(int i = 2; i <= 1_000_000; i++) + { + outputStream.write((",\n" + i).getBytes(StandardCharsets.UTF_8)); + } + outputStream.write("\n]\n".getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(getS3Utils().getAmazonS3(), bucketName, key); + s3UploadOutputStream.write(outputStream.toByteArray(), 0, 5 * 1024 * 1024); + s3UploadOutputStream.write(outputStream.toByteArray(), 0, 3 * 1024 * 1024); + s3UploadOutputStream.write(outputStream.toByteArray(), 0, 3 * 1024 * 1024); + s3UploadOutputStream.close(); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-mongodb/pom.xml b/qqq-backend-module-mongodb/pom.xml index 58ddb23b..4aa23c36 100644 --- a/qqq-backend-module-mongodb/pom.xml +++ b/qqq-backend-module-mongodb/pom.xml @@ -50,11 +50,6 @@ mongodb-driver-sync 4.11.1 - - org.apache.logging.log4j - log4j-slf4j-impl - 2.23.0 - org.testcontainers diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java index fb7d2156..6738f5fc 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java @@ -189,7 +189,7 @@ public class AbstractMongoDBAction Map values = record.getValues(); for(QFieldMetaData field : table.getFields().values()) { - String fieldName = field.getName(); + String fieldName = field.getName(); String fieldBackendName = getFieldBackendName(field); if(fieldBackendName.contains(".")) @@ -405,6 +405,9 @@ public class AbstractMongoDBAction QQueryFilter securityFilter = new QQueryFilter(); securityFilter.setBooleanOperator(QQueryFilter.BooleanOperator.AND); + //////////////////////////////////// + // todo - evolve to use lock tree // + //////////////////////////////////// for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()))) { addSubFilterForRecordSecurityLock(QContext.getQInstance(), QContext.getQSession(), table, securityFilter, recordSecurityLock, null, table.getName(), false); @@ -528,7 +531,6 @@ public class AbstractMongoDBAction /******************************************************************************* ** w/o considering security, just map a QQueryFilter to a Bson searchQuery. *******************************************************************************/ - @SuppressWarnings("checkstyle:Indentation") private Bson makeSearchQueryDocumentWithoutSecurity(QTableMetaData table, QQueryFilter filter) { if(filter == null || !filter.hasAnyCriteria()) @@ -554,7 +556,14 @@ public class AbstractMongoDBAction Serializable value = valueListIterator.next(); if(value instanceof AbstractFilterExpression expression) { - valueListIterator.set(expression.evaluate()); + try + { + valueListIterator.set(expression.evaluate()); + } + catch(QException qe) + { + LOG.warn("Unexpected exception caught evaluating expression", qe); + } } /* todo - is this needed?? @@ -626,6 +635,8 @@ public class AbstractMongoDBAction case IS_NOT_BLANK -> Filters.nor(filterIsBlank(fieldBackendName)); case BETWEEN -> filterBetween(fieldBackendName, values); case NOT_BETWEEN -> Filters.nor(filterBetween(fieldBackendName, values)); + case TRUE -> Filters.or(Filters.eq(fieldBackendName, "true"), Filters.ne(fieldBackendName, "true"), Filters.eq(fieldBackendName, null)); // todo test!! + case FALSE -> Filters.and(Filters.eq(fieldBackendName, "true"), Filters.ne(fieldBackendName, "true"), Filters.eq(fieldBackendName, null)); }); } diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java index fde8026d..c09bea65 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java @@ -70,7 +70,6 @@ public class MongoDBAggregateAction extends AbstractMongoDBAction implements Agg /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") public AggregateOutput execute(AggregateInput aggregateInput) throws QException { MongoClientContainer mongoClientContainer = null; diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java index f9bc56db..3e51e7c2 100644 --- a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java @@ -213,6 +213,33 @@ class MongoDBQueryActionTest extends BaseTest } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testTrueQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.TRUE))); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "'TRUE' query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFalseQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.FALSE))); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "'FALSE' query should find no rows"); + } + + /******************************************************************************* ** diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java index b1bffa96..5a84d73b 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.rdbms.actions.AbstractRDBMSAction; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSAggregateAction; @@ -55,7 +56,10 @@ public class RDBMSBackendModule implements QBackendModuleInterface { private static final QLogger LOG = QLogger.getLogger(RDBMSBackendModule.class); - + static + { + QBackendModuleDispatcher.registerBackendModule(new RDBMSBackendModule()); + } /******************************************************************************* ** Method where a backend module must be able to provide its type (name). diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 5ba71e75..afb3b5d6 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -27,6 +27,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.Duration; import java.time.Instant; import java.time.LocalDate; import java.util.ArrayList; @@ -50,9 +51,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.ImplicitQueryJoinForSecurityLock; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -66,16 +65,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.security.NullValueBehaviorUtil; -import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; -import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; @@ -95,6 +93,9 @@ public abstract class AbstractRDBMSAction protected PreparedStatement statement; protected boolean isCancelled = false; + private static Memoization doesSelectClauseRequireDistinctMemoization = new Memoization() + .withTimeout(Duration.ofDays(365)); + /******************************************************************************* @@ -153,7 +154,7 @@ public abstract class AbstractRDBMSAction if("".equals(value)) { QFieldType type = field.getType(); - if(type.equals(QFieldType.INTEGER) || type.equals(QFieldType.DECIMAL) || type.equals(QFieldType.DATE) || type.equals(QFieldType.DATE_TIME) || type.equals(QFieldType.BOOLEAN)) + if(type.equals(QFieldType.INTEGER) || type.equals(QFieldType.LONG) || type.equals(QFieldType.DECIMAL) || type.equals(QFieldType.DATE) || type.equals(QFieldType.DATE_TIME) || type.equals(QFieldType.BOOLEAN)) { value = null; } @@ -210,7 +211,7 @@ public abstract class AbstractRDBMSAction /******************************************************************************* ** *******************************************************************************/ - protected String makeFromClause(QInstance instance, String tableName, JoinsContext joinsContext) throws QException + protected String makeFromClause(QInstance instance, String tableName, JoinsContext joinsContext, List params) { StringBuilder rs = new StringBuilder(escapeIdentifier(getTableName(instance.getTable(tableName))) + " AS " + escapeIdentifier(tableName)); @@ -227,17 +228,9 @@ public abstract class AbstractRDBMSAction //////////////////////////////////////////////////////////// // find the join in the instance, to set the 'on' clause // //////////////////////////////////////////////////////////// - List joinClauseList = new ArrayList<>(); - String baseTableName = Objects.requireNonNullElse(joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), tableName); - QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> - { - QJoinMetaData found = joinsContext.findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable()); - if(found == null) - { - throw (new RuntimeException("Could not find a join between tables [" + baseTableName + "][" + queryJoin.getJoinTable() + "]")); - } - return (found); - }); + List joinClauseList = new ArrayList<>(); + String baseTableName = Objects.requireNonNullElse(joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), tableName); + QJoinMetaData joinMetaData = Objects.requireNonNull(queryJoin.getJoinMetaData(), () -> "Could not find a join between tables [" + baseTableName + "][" + queryJoin.getJoinTable() + "]"); for(JoinOn joinOn : joinMetaData.getJoinOns()) { @@ -268,6 +261,17 @@ public abstract class AbstractRDBMSAction + " = " + escapeIdentifier(joinTableOrAlias) + "." + escapeIdentifier(getColumnName((rightTable.getField(joinOn.getRightField()))))); } + + if(CollectionUtils.nullSafeHasContents(queryJoin.getSecurityCriteria())) + { + Optional securityOnClause = getSqlWhereStringAndPopulateParamsListFromNonNestedFilter(joinsContext, queryJoin.getSecurityCriteria(), QQueryFilter.BooleanOperator.AND, params); + if(securityOnClause.isPresent()) + { + LOG.debug("Wrote securityOnClause", logPair("clause", securityOnClause)); + joinClauseList.add(securityOnClause.get()); + } + } + rs.append(" ON ").append(StringUtils.join(" AND ", joinClauseList)); } @@ -283,34 +287,66 @@ public abstract class AbstractRDBMSAction *******************************************************************************/ private List sortQueryJoinsForFromClause(String mainTableName, List queryJoins) { + List rs = new ArrayList<>(); + + //////////////////////////////////////////////////////////////////////////////// + // make a copy of the input list that we can feel safe removing elements from // + //////////////////////////////////////////////////////////////////////////////// List inputListCopy = new ArrayList<>(queryJoins); - List rs = new ArrayList<>(); - Set seenTables = new HashSet<>(); - seenTables.add(mainTableName); + /////////////////////////////////////////////////////////////////////////////////////////////////// + // keep track of the tables (or aliases) that we've seen - that's what we'll "grow" outward from // + /////////////////////////////////////////////////////////////////////////////////////////////////// + Set seenTablesOrAliases = new HashSet<>(); + seenTablesOrAliases.add(mainTableName); + //////////////////////////////////////////////////////////////////////////////////// + // loop as long as there are more tables in the inputList, and the keepGoing flag // + // is set (e.g., indicating that we added something in the last iteration) // + //////////////////////////////////////////////////////////////////////////////////// boolean keepGoing = true; while(!inputListCopy.isEmpty() && keepGoing) { keepGoing = false; + Iterator iterator = inputListCopy.iterator(); while(iterator.hasNext()) { - QueryJoin next = iterator.next(); - if((StringUtils.hasContent(next.getBaseTableOrAlias()) && seenTables.contains(next.getBaseTableOrAlias())) || seenTables.contains(next.getJoinTable())) + QueryJoin nextQueryJoin = iterator.next(); + + ////////////////////////////////////////////////////////////////////////// + // get the baseTableOrAlias from this join - and if it isn't set in the // + // QueryJoin, then get it from the left-side of the join's metaData // + ////////////////////////////////////////////////////////////////////////// + String baseTableOrAlias = nextQueryJoin.getBaseTableOrAlias(); + if(baseTableOrAlias == null && nextQueryJoin.getJoinMetaData() != null) { - rs.add(next); - if(StringUtils.hasContent(next.getBaseTableOrAlias())) + baseTableOrAlias = nextQueryJoin.getJoinMetaData().getLeftTable(); + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we have a baseTableOrAlias (would we ever not?), and we've seen it before - OR - we've seen this query join's joinTableOrAlias, // + // then we can add this pair of namesOrAliases to our seen-set, remove this queryJoin from the inputListCopy (iterator), and keep going // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if((StringUtils.hasContent(baseTableOrAlias) && seenTablesOrAliases.contains(baseTableOrAlias)) || seenTablesOrAliases.contains(nextQueryJoin.getJoinTableOrItsAlias())) + { + rs.add(nextQueryJoin); + if(StringUtils.hasContent(baseTableOrAlias)) { - seenTables.add(next.getBaseTableOrAlias()); + seenTablesOrAliases.add(baseTableOrAlias); } - seenTables.add(next.getJoinTable()); + + seenTablesOrAliases.add(nextQueryJoin.getJoinTableOrItsAlias()); iterator.remove(); keepGoing = true; } } } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // in case any are left, add them all here - does this ever happen? // + // the only time a conditional breakpoint here fires in the RDBMS test suite, is in query designed to throw. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// rs.addAll(inputListCopy); return (rs); @@ -319,208 +355,72 @@ public abstract class AbstractRDBMSAction /******************************************************************************* - ** method that sub-classes should call to make a full WHERE clause, including - ** security clauses. + ** Method to make a full WHERE clause. + ** + ** Note that criteria for security are assumed to have been added to the filter + ** during the construction of the JoinsContext. *******************************************************************************/ - protected String makeWhereClause(QInstance instance, QSession session, QTableMetaData table, JoinsContext joinsContext, QQueryFilter filter, List params) throws IllegalArgumentException, QException - { - String whereClauseWithoutSecurity = makeWhereClauseWithoutSecurity(instance, table, joinsContext, filter, params); - QQueryFilter securityFilter = getSecurityFilter(instance, session, table, joinsContext); - if(!securityFilter.hasAnyCriteria()) - { - return (whereClauseWithoutSecurity); - } - String securityWhereClause = makeWhereClauseWithoutSecurity(instance, table, joinsContext, securityFilter, params); - return ("(" + whereClauseWithoutSecurity + ") AND (" + securityWhereClause + ")"); - } - - - - /******************************************************************************* - ** private method for making the part of a where clause that gets AND'ed to the - ** security clause. Recursively handles sub-clauses. - *******************************************************************************/ - private String makeWhereClauseWithoutSecurity(QInstance instance, QTableMetaData table, JoinsContext joinsContext, QQueryFilter filter, List params) throws IllegalArgumentException, QException + protected String makeWhereClause(JoinsContext joinsContext, QQueryFilter filter, List params) throws IllegalArgumentException { if(filter == null || !filter.hasAnyCriteria()) { return ("1 = 1"); } - String clause = getSqlWhereStringAndPopulateParamsListFromNonNestedFilter(instance, table, joinsContext, filter.getCriteria(), filter.getBooleanOperator(), params); + Optional clause = getSqlWhereStringAndPopulateParamsListFromNonNestedFilter(joinsContext, filter.getCriteria(), filter.getBooleanOperator(), params); if(!CollectionUtils.nullSafeHasContents(filter.getSubFilters())) { /////////////////////////////////////////////////////////////// // if there are no sub-clauses, then just return this clause // + // and if there's no clause, use the default 1 = 1 // /////////////////////////////////////////////////////////////// - return (clause); + return (clause.orElse("1 = 1")); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // else, build a list of clauses - recursively expanding the sub-filters into clauses, then return them joined with our operator // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// List clauses = new ArrayList<>(); - if(StringUtils.hasContent(clause)) + if(clause.isPresent() && StringUtils.hasContent(clause.get())) { - clauses.add("(" + clause + ")"); + clauses.add("(" + clause.get() + ")"); } + for(QQueryFilter subFilter : filter.getSubFilters()) { - String subClause = makeWhereClauseWithoutSecurity(instance, table, joinsContext, subFilter, params); + String subClause = makeWhereClause(joinsContext, subFilter, params); if(StringUtils.hasContent(subClause)) { clauses.add("(" + subClause + ")"); } } + return (String.join(" " + filter.getBooleanOperator().toString() + " ", clauses)); } - /******************************************************************************* - ** Build a QQueryFilter to apply record-level security to the query. - ** Note, it may be empty, if there are no lock fields, or all are all-access. - *******************************************************************************/ - private QQueryFilter getSecurityFilter(QInstance instance, QSession session, QTableMetaData table, JoinsContext joinsContext) - { - QQueryFilter securityFilter = new QQueryFilter(); - securityFilter.setBooleanOperator(QQueryFilter.BooleanOperator.AND); - - for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()))) - { - // todo - uh, if it's a RIGHT (or FULL) join, then, this should be isOuter = true, right? - boolean isOuter = false; - addSubFilterForRecordSecurityLock(instance, session, table, securityFilter, recordSecurityLock, joinsContext, table.getName(), isOuter); - } - - for(QueryJoin queryJoin : CollectionUtils.nonNullList(joinsContext.getQueryJoins())) - { - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // for user-added joins, we want to add their security-locks to the query // - // but if a join was implicitly added because it's needed to find a security lock on table being queried, // - // don't add additional layers of locks for each join table. that's the idea here at least. // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(queryJoin instanceof ImplicitQueryJoinForSecurityLock) - { - continue; - } - - QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); - for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks()))) - { - boolean isOuter = queryJoin.getType().equals(QueryJoin.Type.LEFT); // todo full? - addSubFilterForRecordSecurityLock(instance, session, joinTable, securityFilter, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias(), isOuter); - } - } - - return (securityFilter); - } - - - /******************************************************************************* ** + ** @return optional sql where sub-clause, as in "x AND y" *******************************************************************************/ - private static void addSubFilterForRecordSecurityLock(QInstance instance, QSession session, QTableMetaData table, QQueryFilter securityFilter, RecordSecurityLock recordSecurityLock, JoinsContext joinsContext, String tableNameOrAlias, boolean isOuter) - { - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // check if the key type has an all-access key, and if so, if it's set to true for the current user/session // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); - if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName())) - { - if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN)) - { - /////////////////////////////////////////////////////////////////////////////// - // if we have all-access on this key, then we don't need a criterion for it. // - /////////////////////////////////////////////////////////////////////////////// - return; - } - } - - String fieldName = tableNameOrAlias + "." + recordSecurityLock.getFieldName(); - if(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain())) - { - fieldName = recordSecurityLock.getFieldName(); - } - - /////////////////////////////////////////////////////////////////////////////////////////// - // else - get the key values from the session and decide what kind of criterion to build // - /////////////////////////////////////////////////////////////////////////////////////////// - QQueryFilter lockFilter = new QQueryFilter(); - List lockCriteria = new ArrayList<>(); - lockFilter.setCriteria(lockCriteria); - - QFieldType type = QFieldType.INTEGER; - try - { - JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(fieldName); - type = fieldAndTableNameOrAlias.field().getType(); - } - catch(Exception e) - { - LOG.debug("Error getting field type... Trying Integer", e); - } - - List securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type); - if(CollectionUtils.nullSafeIsEmpty(securityKeyValues)) - { - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // handle user with no values -- they can only see null values, and only iff the lock's null-value behavior is ALLOW // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock))) - { - lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK)); - } - else - { - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // else, if no user/session values, and null-value behavior is deny, then setup a FALSE condition, to allow no rows. // - // todo - make some explicit contradiction here - maybe even avoid running the whole query - as you're not allowed ANY records // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, Collections.emptyList())); - } - } - else - { - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // else, if user/session has some values, build an IN rule - // - // noting that if the lock's null-value behavior is ALLOW, then we actually want IS_NULL_OR_IN, not just IN // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock))) - { - lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_NULL_OR_IN, securityKeyValues)); - } - else - { - lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, securityKeyValues)); - } - } - - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if this field is on the outer side of an outer join, then if we do a straight filter on it, then we're basically // - // nullifying the outer join... so for an outer join use-case, OR the security field criteria with a primary-key IS NULL // - // which will make missing rows from the join be found. // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(isOuter) - { - lockFilter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); - lockFilter.addCriteria(new QFilterCriteria(tableNameOrAlias + "." + table.getPrimaryKeyField(), QCriteriaOperator.IS_BLANK)); - } - - securityFilter.addSubFilter(lockFilter); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private String getSqlWhereStringAndPopulateParamsListFromNonNestedFilter(QInstance instance, QTableMetaData table, JoinsContext joinsContext, List criteria, QQueryFilter.BooleanOperator booleanOperator, List params) throws IllegalArgumentException + private Optional getSqlWhereStringAndPopulateParamsListFromNonNestedFilter(JoinsContext joinsContext, List criteria, QQueryFilter.BooleanOperator booleanOperator, List params) throws IllegalArgumentException { List clauses = new ArrayList<>(); for(QFilterCriteria criterion : criteria) { + if(criterion.getFieldName() == null) + { + LOG.info("QFilter criteria is missing a fieldName - will not be included in query."); + continue; + } + + if(criterion.getOperator() == null) + { + LOG.info("QFilter criteria is missing a operator - will not be included in query.", logPair("fieldName", criterion.getFieldName())); + continue; + } + JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(criterion.getFieldName()); List values = criterion.getValues() == null ? new ArrayList<>() : new ArrayList<>(criterion.getValues()); @@ -530,25 +430,22 @@ public abstract class AbstractRDBMSAction Integer expectedNoOfParams = null; switch(criterion.getOperator()) { - case EQUALS: + case EQUALS -> { clause += " = ?"; expectedNoOfParams = 1; - break; } - case NOT_EQUALS: + case NOT_EQUALS -> { clause += " != ?"; expectedNoOfParams = 1; - break; } - case NOT_EQUALS_OR_IS_NULL: + case NOT_EQUALS_OR_IS_NULL -> { clause += " != ? OR " + column + " IS NULL "; expectedNoOfParams = 1; - break; } - case IN: + case IN -> { if(values.isEmpty()) { @@ -561,9 +458,8 @@ public abstract class AbstractRDBMSAction { clause += " IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; } - break; } - case IS_NULL_OR_IN: + case IS_NULL_OR_IN -> { clause += " IS NULL "; @@ -571,9 +467,8 @@ public abstract class AbstractRDBMSAction { clause += " OR " + column + " IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; } - break; } - case NOT_IN: + case NOT_IN -> { if(values.isEmpty()) { @@ -586,87 +481,74 @@ public abstract class AbstractRDBMSAction { clause += " NOT IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; } - break; } - case LIKE: + case LIKE -> { clause += " LIKE ?"; expectedNoOfParams = 1; - break; } - case NOT_LIKE: + case NOT_LIKE -> { clause += " NOT LIKE ?"; expectedNoOfParams = 1; - break; } - case STARTS_WITH: + case STARTS_WITH -> { clause += " LIKE ?"; ActionHelper.editFirstValue(values, (s -> s + "%")); expectedNoOfParams = 1; - break; } - case ENDS_WITH: + case ENDS_WITH -> { clause += " LIKE ?"; ActionHelper.editFirstValue(values, (s -> "%" + s)); expectedNoOfParams = 1; - break; } - case CONTAINS: + case CONTAINS -> { clause += " LIKE ?"; ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); expectedNoOfParams = 1; - break; } - case NOT_STARTS_WITH: + case NOT_STARTS_WITH -> { clause += " NOT LIKE ?"; ActionHelper.editFirstValue(values, (s -> s + "%")); expectedNoOfParams = 1; - break; } - case NOT_ENDS_WITH: + case NOT_ENDS_WITH -> { clause += " NOT LIKE ?"; ActionHelper.editFirstValue(values, (s -> "%" + s)); expectedNoOfParams = 1; - break; } - case NOT_CONTAINS: + case NOT_CONTAINS -> { clause += " NOT LIKE ?"; ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); expectedNoOfParams = 1; - break; } - case LESS_THAN: + case LESS_THAN -> { clause += " < ?"; expectedNoOfParams = 1; - break; } - case LESS_THAN_OR_EQUALS: + case LESS_THAN_OR_EQUALS -> { clause += " <= ?"; expectedNoOfParams = 1; - break; } - case GREATER_THAN: + case GREATER_THAN -> { clause += " > ?"; expectedNoOfParams = 1; - break; } - case GREATER_THAN_OR_EQUALS: + case GREATER_THAN_OR_EQUALS -> { clause += " >= ?"; expectedNoOfParams = 1; - break; } - case IS_BLANK: + case IS_BLANK -> { clause += " IS NULL"; if(field.getType().isStringLike()) @@ -674,9 +556,8 @@ public abstract class AbstractRDBMSAction clause += " OR " + column + " = ''"; } expectedNoOfParams = 0; - break; } - case IS_NOT_BLANK: + case IS_NOT_BLANK -> { clause += " IS NOT NULL"; if(field.getType().isStringLike()) @@ -684,24 +565,28 @@ public abstract class AbstractRDBMSAction clause += " AND " + column + " != ''"; } expectedNoOfParams = 0; - break; } - case BETWEEN: + case BETWEEN -> { clause += " BETWEEN ? AND ?"; expectedNoOfParams = 2; - break; } - case NOT_BETWEEN: + case NOT_BETWEEN -> { clause += " NOT BETWEEN ? AND ?"; expectedNoOfParams = 2; - break; } - default: + case TRUE -> { - throw new IllegalArgumentException("Unexpected operator: " + criterion.getOperator()); + clause = " 1 = 1 "; + expectedNoOfParams = 0; } + case FALSE -> + { + clause = " 0 = 1 "; + expectedNoOfParams = 0; + } + default -> throw new IllegalStateException("Unexpected operator: " + criterion.getOperator()); } if(expectedNoOfParams != null) @@ -733,7 +618,14 @@ public abstract class AbstractRDBMSAction Serializable value = valueListIterator.next(); if(value instanceof AbstractFilterExpression expression) { - valueListIterator.set(expression.evaluate()); + try + { + valueListIterator.set(expression.evaluate()); + } + catch(QException qe) + { + LOG.warn("Unexpected exception caught evaluating expression", qe); + } } else { @@ -756,7 +648,16 @@ public abstract class AbstractRDBMSAction params.addAll(values); } - return (String.join(" " + booleanOperator.toString() + " ", clauses)); + ////////////////////////////////////////////////////////////////////////////// + // since we're skipping criteria w/o a field or operator in the loop - // + // we can get to the end here without any clauses... so, return a null here // + ////////////////////////////////////////////////////////////////////////////// + if(clauses.isEmpty()) + { + return (Optional.empty()); + } + + return (Optional.of(String.join(" " + booleanOperator.toString() + " ", clauses))); } @@ -853,6 +754,10 @@ public abstract class AbstractRDBMSAction { return (QueryManager.getInteger(resultSet, i)); } + case LONG: + { + return (QueryManager.getLong(resultSet, i)); + } case DECIMAL: { return (QueryManager.getBigDecimal(resultSet, i)); @@ -950,6 +855,18 @@ public abstract class AbstractRDBMSAction + /******************************************************************************* + ** Make it easy (e.g., for tests) to turn on logging of SQL + *******************************************************************************/ + public static void setLogSQL(boolean on, boolean doReformat, String loggerOrSystemOut) + { + setLogSQL(on); + setLogSQLOutput(loggerOrSystemOut); + setLogSQLReformat(doReformat); + } + + + /******************************************************************************* ** Make it easy (e.g., for tests) to turn on logging of SQL *******************************************************************************/ @@ -1036,25 +953,52 @@ public abstract class AbstractRDBMSAction /******************************************************************************* ** method that looks at security lock joins, and if a one-to-many is found where ** the specified field name is on the 'right side' of the join, then a distinct - ** needs added to select clause + ** needs added to select clause. + ** + ** Memoized because it's a lot of gyrations, and it never ever changes for a + ** running server. *******************************************************************************/ protected boolean doesSelectClauseRequireDistinct(QTableMetaData table) { - if(table != null) + if(table == null) { - for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()))) + return (false); + } + + return doesSelectClauseRequireDistinctMemoization.getResult(table.getName(), (name) -> + { + MultiRecordSecurityLock multiRecordSecurityLock = RecordSecurityLockFilters.filterForReadLockTree(CollectionUtils.nonNullList(table.getRecordSecurityLocks())); + return doesMultiLockRequireDistinct(multiRecordSecurityLock, table); + }).orElse(false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean doesMultiLockRequireDistinct(MultiRecordSecurityLock multiRecordSecurityLock, QTableMetaData table) + { + for(RecordSecurityLock recordSecurityLock : multiRecordSecurityLock.getLocks()) + { + if(recordSecurityLock instanceof MultiRecordSecurityLock childMultiLock) { - for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain())) + if(doesMultiLockRequireDistinct(childMultiLock, table)) { - QJoinMetaData joinMetaData = QContext.getQInstance().getJoin(joinName); - if(JoinType.ONE_TO_MANY.equals(joinMetaData.getType()) && !joinMetaData.getRightTable().equals(table.getName())) - { - return (true); - } - else if(JoinType.MANY_TO_ONE.equals(joinMetaData.getType()) && !joinMetaData.getLeftTable().equals(table.getName())) - { - return (true); - } + return (true); + } + } + + for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain())) + { + QJoinMetaData joinMetaData = QContext.getQInstance().getJoin(joinName); + if(JoinType.ONE_TO_MANY.equals(joinMetaData.getType()) && !joinMetaData.getRightTable().equals(table.getName())) + { + return (true); + } + else if(JoinType.MANY_TO_ONE.equals(joinMetaData.getType()) && !joinMetaData.getLeftTable().equals(table.getName())) + { + return (true); } } } @@ -1129,4 +1073,20 @@ public abstract class AbstractRDBMSAction } } + + + /******************************************************************************* + ** Either clone the input filter (so we can change it safely), or return a new blank filter. + *******************************************************************************/ + protected QQueryFilter clonedOrNewFilter(QQueryFilter filter) + { + if(filter == null) + { + return (new QQueryFilter()); + } + else + { + return (filter.clone()); + } + } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index ce720120..21a2052f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -59,6 +59,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega private ActionTimeoutHelper actionTimeoutHelper; + /******************************************************************************* ** *******************************************************************************/ @@ -68,16 +69,17 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega { QTableMetaData table = aggregateInput.getTable(); - JoinsContext joinsContext = new JoinsContext(aggregateInput.getInstance(), table.getName(), aggregateInput.getQueryJoins(), aggregateInput.getFilter()); - String fromClause = makeFromClause(aggregateInput.getInstance(), table.getName(), joinsContext); + QQueryFilter filter = clonedOrNewFilter(aggregateInput.getFilter()); + JoinsContext joinsContext = new JoinsContext(aggregateInput.getInstance(), table.getName(), aggregateInput.getQueryJoins(), filter); + + List params = new ArrayList<>(); + + String fromClause = makeFromClause(aggregateInput.getInstance(), table.getName(), joinsContext, params); List selectClauses = buildSelectClauses(aggregateInput, joinsContext); String sql = "SELECT " + StringUtils.join(", ", selectClauses) - + " FROM " + fromClause; - - QQueryFilter filter = aggregateInput.getFilter(); - List params = new ArrayList<>(); - sql += " WHERE " + makeWhereClause(aggregateInput.getInstance(), aggregateInput.getSession(), table, joinsContext, filter, params); + + " FROM " + fromClause + + " WHERE " + makeWhereClause(joinsContext, filter, params); if(CollectionUtils.nullSafeHasContents(aggregateInput.getGroupBys())) { @@ -143,7 +145,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega QFieldType fieldType = aggregate.getFieldType(); if(fieldType == null) { - if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG))) + if((field.getType().equals(QFieldType.INTEGER) || field.getType().equals(QFieldType.LONG)) && (aggregate.getOperator().equals(AggregateOperator.AVG))) { fieldType = QFieldType.DECIMAL; } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index 7177a4a1..ba167674 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -62,7 +62,8 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf { QTableMetaData table = countInput.getTable(); - JoinsContext joinsContext = new JoinsContext(countInput.getInstance(), countInput.getTableName(), countInput.getQueryJoins(), countInput.getFilter()); + QQueryFilter filter = clonedOrNewFilter(countInput.getFilter()); + JoinsContext joinsContext = new JoinsContext(countInput.getInstance(), countInput.getTableName(), countInput.getQueryJoins(), filter); JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(table.getPrimaryKeyField()); boolean requiresDistinct = doesSelectClauseRequireDistinct(table); @@ -74,12 +75,10 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf clausePrefix = "SELECT COUNT(DISTINCT (" + primaryKeyColumn + ")) AS distinct_count, COUNT(*)"; } - String sql = clausePrefix + " AS record_count FROM " - + makeFromClause(countInput.getInstance(), table.getName(), joinsContext); - - QQueryFilter filter = countInput.getFilter(); List params = new ArrayList<>(); - sql += " WHERE " + makeWhereClause(countInput.getInstance(), countInput.getSession(), table, joinsContext, filter, params); + String sql = clausePrefix + " AS record_count " + + " FROM " + makeFromClause(countInput.getInstance(), table.getName(), joinsContext, params) + + " WHERE " + makeWhereClause(joinsContext, filter, params); // todo sql customization - can edit sql and/or param list setSqlAndJoinsInQueryStat(sql, joinsContext); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java index 734c3202..baec4f0a 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java @@ -268,7 +268,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte String tableName = getTableName(table); JoinsContext joinsContext = new JoinsContext(deleteInput.getInstance(), table.getName(), new ArrayList<>(), deleteInput.getQueryFilter()); - String whereClause = makeWhereClause(deleteInput.getInstance(), deleteInput.getSession(), table, joinsContext, filter, params); + String whereClause = makeWhereClause(joinsContext, filter, params); // todo sql customization - can edit sql and/or param list? String sql = "DELETE FROM " diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 698a2053..86f119fb 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -93,13 +93,12 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf StringBuilder sql = new StringBuilder(makeSelectClause(queryInput)); - JoinsContext joinsContext = new JoinsContext(queryInput.getInstance(), tableName, queryInput.getQueryJoins(), queryInput.getFilter()); - sql.append(" FROM ").append(makeFromClause(queryInput.getInstance(), tableName, joinsContext)); + QQueryFilter filter = clonedOrNewFilter(queryInput.getFilter()); + JoinsContext joinsContext = new JoinsContext(queryInput.getInstance(), tableName, queryInput.getQueryJoins(), filter); - QQueryFilter filter = queryInput.getFilter(); List params = new ArrayList<>(); - - sql.append(" WHERE ").append(makeWhereClause(queryInput.getInstance(), queryInput.getSession(), table, joinsContext, filter, params)); + sql.append(" FROM ").append(makeFromClause(queryInput.getInstance(), tableName, joinsContext, params)); + sql.append(" WHERE ").append(makeWhereClause(joinsContext, filter, params)); if(filter != null && CollectionUtils.nullSafeHasContents(filter.getOrderBys())) { @@ -357,16 +356,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf *******************************************************************************/ private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException { - if(mysqlResultSetOptimizationEnabled && connection.getClass().getName().startsWith("com.mysql")) + if(connection.getClass().getName().startsWith("com.mysql")) { - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html // - // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // - // with this change, we start to get results immediately, and the total runtime also seems lower... // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - statement.setFetchSize(Integer.MIN_VALUE); - return (statement); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're allowed to use the mysqlResultSetOptimization, and we have the query hint of "expected large result set", then do it. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(mysqlResultSetOptimizationEnabled && queryInput.getQueryHints() != null && queryInput.getQueryHints().contains(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS)) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html // + // without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). // + // with this change, we start to get results immediately, and the total runtime also seems lower... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + statement.setFetchSize(Integer.MIN_VALUE); + return (statement); + } } return (connection.prepareStatement(sql)); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java new file mode 100644 index 00000000..7fc556d4 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java @@ -0,0 +1,209 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; + + +/******************************************************************************* + ** note - this class is pretty mysql-specific + *******************************************************************************/ +public class RDBMSTableMetaDataBuilder +{ + private static final QLogger LOG = QLogger.getLogger(RDBMSTableMetaDataBuilder.class); + + private boolean useCamelCaseNames = true; + + private static Map typeMap = new HashMap<>(); + + static + { + //////////////////////////////////////////////// + // these are types as returned by mysql // + // let null in here mean unsupported QQQ type // + //////////////////////////////////////////////// + typeMap.put("TEXT", QFieldType.TEXT); + typeMap.put("BINARY", QFieldType.BLOB); + typeMap.put("SET", null); + typeMap.put("VARBINARY", QFieldType.BLOB); + typeMap.put("MEDIUMBLOB", QFieldType.BLOB); + typeMap.put("NUMERIC", QFieldType.INTEGER); + typeMap.put("INTEGER", QFieldType.INTEGER); + typeMap.put("BIGINT UNSIGNED", QFieldType.LONG); + typeMap.put("MEDIUMINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("SMALLINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("TINYINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("BIT", null); + typeMap.put("FLOAT", null); + typeMap.put("REAL", null); + typeMap.put("VARCHAR", QFieldType.STRING); + typeMap.put("BOOL", QFieldType.BOOLEAN); + typeMap.put("YEAR", null); + typeMap.put("TIME", QFieldType.TIME); + typeMap.put("TIMESTAMP", QFieldType.DATE_TIME); + } + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData buildTableMetaData(RDBMSBackendMetaData backendMetaData, String tableName) throws QException + { + try(Connection connection = new ConnectionManager().getConnection(backendMetaData)) + { + List fieldMetaDataList = new ArrayList<>(); + String primaryKey = null; + + DatabaseMetaData databaseMetaData = connection.getMetaData(); + Map dataTypeMap = new HashMap<>(); + ResultSet typeInfoResultSet = databaseMetaData.getTypeInfo(); + while(typeInfoResultSet.next()) + { + String name = typeInfoResultSet.getString("TYPE_NAME"); + Integer id = typeInfoResultSet.getInt("DATA_TYPE"); + dataTypeMap.put(id, name); + } + + // todo - for h2, uppercase both db & table names... + String databaseName = backendMetaData.getDatabaseName(); // these work for mysql - unclear about other vendors. + String schemaName = null; + String tableNameForMetaDataQueries = tableName; + + if(backendMetaData.getVendor().equals("h2")) + { + databaseName = databaseName.toUpperCase(); + tableNameForMetaDataQueries = tableName.toUpperCase(); + } + + try(ResultSet tableResultSet = databaseMetaData.getTables(databaseName, schemaName, tableNameForMetaDataQueries, null)) + { + if(!tableResultSet.next()) + { + throw (new QException("Table: " + tableName + " was not found in backend: " + backendMetaData.getName())); + } + } + + try(ResultSet columnsResultSet = databaseMetaData.getColumns(databaseName, schemaName, tableNameForMetaDataQueries, null)) + { + while(columnsResultSet.next()) + { + String columnName = columnsResultSet.getString("COLUMN_NAME"); + String columnSize = columnsResultSet.getString("COLUMN_SIZE"); + Integer dataTypeId = columnsResultSet.getInt("DATA_TYPE"); + String isNullable = columnsResultSet.getString("IS_NULLABLE"); + String isAutoIncrement = columnsResultSet.getString("IS_AUTOINCREMENT"); + + String dataTypeName = dataTypeMap.get(dataTypeId); + QFieldType type = typeMap.get(dataTypeName); + if(type == null) + { + LOG.info("Table " + tableName + " column " + columnName + " has an unmapped type: " + dataTypeId + ". Field will not be added to QTableMetaData"); + continue; + } + + String qqqFieldName = QInstanceEnricher.inferNameFromBackendName(columnName); + + QFieldMetaData fieldMetaData = new QFieldMetaData(qqqFieldName, type) + // todo - what string? .withIsRequired(!isNullable) + .withBackendName(columnName); + + fieldMetaDataList.add(fieldMetaData); + + if("YES".equals(isAutoIncrement)) + { + primaryKey = qqqFieldName; + } + } + } + + if(fieldMetaDataList.isEmpty()) + { + throw (new QException("Could not find any usable fields in table: " + tableName)); + } + + if(primaryKey == null) + { + throw (new QException("Could not find primary key in table: " + tableName)); + } + + String qqqTableName = QInstanceEnricher.inferNameFromBackendName(tableName); + + QTableMetaData tableMetaData = new QTableMetaData() + .withBackendName(backendMetaData.getName()) + .withName(qqqTableName) + .withBackendDetails(new RDBMSTableBackendDetails().withTableName(tableName)) + .withFields(fieldMetaDataList) + .withPrimaryKeyField(primaryKey); + + return (tableMetaData); + } + catch(Exception e) + { + throw (new QException("Error automatically building table meta data for table: " + tableName, e)); + } + } + + + /******************************************************************************* + ** Getter for useCamelCaseNames + *******************************************************************************/ + public boolean getUseCamelCaseNames() + { + return (this.useCamelCaseNames); + } + + + + /******************************************************************************* + ** Setter for useCamelCaseNames + *******************************************************************************/ + public void setUseCamelCaseNames(boolean useCamelCaseNames) + { + this.useCamelCaseNames = useCamelCaseNames; + } + + + + /******************************************************************************* + ** Fluent setter for useCamelCaseNames + *******************************************************************************/ + public RDBMSTableMetaDataBuilder withUseCamelCaseNames(boolean useCamelCaseNames) + { + this.useCamelCaseNames = useCamelCaseNames; + return (this); + } + + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionWithinRDBMSTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionWithinRDBMSTest.java new file mode 100644 index 00000000..1b9671cc --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionWithinRDBMSTest.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.io.ByteArrayOutputStream; +import java.util.List; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Test some harder exports, using RDBMS backend. + *******************************************************************************/ +public class ExportActionWithinRDBMSTest extends RDBMSActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + super.primeTestDatabase(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIncludingFieldsFromExposedJoinTableWithTwoJoinsToMainTable() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ExportInput exportInput = new ExportInput(); + exportInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + exportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.CSV) + .withReportOutputStream(baos)); + exportInput.setQueryFilter(new QQueryFilter()); + exportInput.setFieldNames(List.of("id", "storeId", "billToPersonId", "currentOrderInstructionsId", TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS + ".id", TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS + ".instructions")); + ExportOutput exportOutput = new ExportAction().execute(exportInput); + + assertNotNull(exportOutput); + + /////////////////////////////////////////////////////////////////////////// + // if there was an exception running the query, we get back 0 records... // + /////////////////////////////////////////////////////////////////////////// + assertEquals(3, exportOutput.getRecordCount()); + } + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index d882807a..2570f6ea 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; @@ -61,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; public class TestUtils { public static final String DEFAULT_BACKEND_NAME = "default"; + public static final String MEMORY_BACKEND_NAME = "memory"; public static final String TABLE_NAME_PERSON = "personTable"; public static final String TABLE_NAME_PERSONAL_ID_CARD = "personalIdCard"; @@ -107,6 +110,7 @@ public class TestUtils { QInstance qInstance = new QInstance(); qInstance.addBackend(defineBackend()); + qInstance.addBackend(defineMemoryBackend()); qInstance.addTable(defineTablePerson()); qInstance.addPossibleValueSource(definePvsPerson()); qInstance.addTable(defineTablePersonalIdCard()); @@ -118,6 +122,18 @@ public class TestUtils + /******************************************************************************* + ** Define the in-memory backend used in standard tests + *******************************************************************************/ + public static QBackendMetaData defineMemoryBackend() + { + return new QBackendMetaData() + .withName(MEMORY_BACKEND_NAME) + .withBackendType(MemoryBackendModule.class); + } + + + /******************************************************************************* ** Define the authentication used in standard tests - using 'mock' type. ** @@ -243,6 +259,7 @@ public class TestUtils .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine")) .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem"))) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")).withLabel("Current Order Instructions")) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java index 6a3cb3a8..74d8804e 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java @@ -161,10 +161,10 @@ public class RDBMSInsertActionTest extends RDBMSActionTest insertInput.setRecords(List.of( new QRecord().withValue("storeId", 1).withValue("billToPersonId", 100).withValue("shipToPersonId", 200) - .withAssociatedRecord("orderLine", new QRecord().withValue("sku", "BASIC1").withValue("quantity", 1) + .withAssociatedRecord("orderLine", new QRecord().withValue("storeId", 1).withValue("sku", "BASIC1").withValue("quantity", 1) .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-1.1").withValue("value", "LINE-VAL-1"))) - .withAssociatedRecord("orderLine", new QRecord().withValue("sku", "BASIC2").withValue("quantity", 2) + .withAssociatedRecord("orderLine", new QRecord().withValue("storeId", 1).withValue("sku", "BASIC2").withValue("quantity", 2) .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-2.1").withValue("value", "LINE-VAL-2")) .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-2.2").withValue("value", "LINE-VAL-3"))) )); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java new file mode 100644 index 00000000..12993de3 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java @@ -0,0 +1,1032 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.actions; + + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Tests on RDBMS - specifically dealing with Joins. + *******************************************************************************/ +public class RDBMSQueryActionJoinsTest extends RDBMSActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + super.primeTestDatabase(); + + AbstractRDBMSAction.setLogSQL(true); + AbstractRDBMSAction.setLogSQLReformat(true); + AbstractRDBMSAction.setLogSQLOutput("system.out"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + AbstractRDBMSAction.setLogSQL(false); + AbstractRDBMSAction.setLogSQLReformat(false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryInput initQueryRequest() + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_PERSON); + return queryInput; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFilterFromJoinTableImplicitly() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("personalIdCard.idNumber", QCriteriaOperator.EQUALS, "19800531"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Query should find 1 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneInnerJoinWithoutWhere() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Join query should find 3 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("personalIdCard.idNumber").equals("19800531")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("personalIdCard.idNumber").equals("19800515")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("personalIdCard.idNumber").equals("19760528")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneLeftJoinWithoutWhere() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withType(QueryJoin.Type.LEFT).withSelect(true)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Left Join query should find 5 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("personalIdCard.idNumber").equals("19800531")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("personalIdCard.idNumber").equals("19800515")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("personalIdCard.idNumber").equals("19760528")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Garret") && r.getValue("personalIdCard.idNumber") == null); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tyler") && r.getValue("personalIdCard.idNumber") == null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneRightJoinWithoutWhere() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withType(QueryJoin.Type.RIGHT).withSelect(true)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(6, queryOutput.getRecords().size(), "Right Join query should find 6 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("personalIdCard.idNumber").equals("19800531")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("personalIdCard.idNumber").equals("19800515")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("personalIdCard.idNumber").equals("19760528")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValue("firstName") == null && r.getValueString("personalIdCard.idNumber").equals("123123123")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValue("firstName") == null && r.getValueString("personalIdCard.idNumber").equals("987987987")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValue("firstName") == null && r.getValueString("personalIdCard.idNumber").equals("456456456")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneInnerJoinWithWhere() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true)); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber", QCriteriaOperator.STARTS_WITH, "1980"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Join query should find 2 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("personalIdCard.idNumber").equals("19800531")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("personalIdCard.idNumber").equals("19800515")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneInnerJoinWithOrderBy() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QueryInput queryInput = initQueryRequest(); + queryInput.withQueryJoin(new QueryJoin(qInstance.getJoin(TestUtils.TABLE_NAME_PERSON + "Join" + StringUtils.ucFirst(TestUtils.TABLE_NAME_PERSONAL_ID_CARD))).withSelect(true)); + queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Join query should find 3 rows"); + List idNumberListFromQuery = queryOutput.getRecords().stream().map(r -> r.getValueString(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber")).toList(); + assertEquals(List.of("19760528", "19800515", "19800531"), idNumberListFromQuery); + + ///////////////////////// + // repeat, sorted desc // + ///////////////////////// + queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber", false))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Join query should find 3 rows"); + idNumberListFromQuery = queryOutput.getRecords().stream().map(r -> r.getValueString(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber")).toList(); + assertEquals(List.of("19800531", "19800515", "19760528"), idNumberListFromQuery); + } + + + + /******************************************************************************* + ** In the prime data, we've got 1 order line set up with an item from a different + ** store than its order. Write a query to find such a case. + *******************************************************************************/ + @Test + void testFiveTableOmsJoinFindMismatchedStoreId() throws Exception + { + QueryInput queryInput = new QueryInput(); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER, TestUtils.TABLE_NAME_STORE).withAlias("orderStore").withSelect(true)); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER, TestUtils.TABLE_NAME_ORDER_LINE).withSelect(true)); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_LINE, TestUtils.TABLE_NAME_ITEM).withSelect(true)); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ITEM, TestUtils.TABLE_NAME_STORE).withAlias("itemStore").withSelect(true)); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria().withFieldName("orderStore.id").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("item.storeId"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); + QRecord qRecord = queryOutput.getRecords().get(0); + assertEquals(2, qRecord.getValueInteger("id")); + assertEquals(1, qRecord.getValueInteger("orderStore.id")); + assertEquals(2, qRecord.getValueInteger("itemStore.id")); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // run the same setup, but this time, use the other-field-name as itemStore.id, instead of item.storeId // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + queryInput.setFilter(new QQueryFilter(new QFilterCriteria().withFieldName("orderStore.id").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("itemStore.id"))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); + qRecord = queryOutput.getRecords().get(0); + assertEquals(2, qRecord.getValueInteger("id")); + assertEquals(1, qRecord.getValueInteger("orderStore.id")); + assertEquals(2, qRecord.getValueInteger("itemStore.id")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOmsQueryByOrderLines() throws Exception + { + AtomicInteger orderLineCount = new AtomicInteger(); + runTestSql("SELECT COUNT(*) from order_line", (rs) -> + { + rs.next(); + orderLineCount.set(rs.getInt(1)); + }); + + QueryInput queryInput = new QueryInput(); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER_LINE); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER).withSelect(true)); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(orderLineCount.get(), queryOutput.getRecords().size(), "# of rows found by query"); + assertEquals(3, queryOutput.getRecords().stream().filter(r -> r.getValueInteger("order.id").equals(1)).count()); + assertEquals(1, queryOutput.getRecords().stream().filter(r -> r.getValueInteger("order.id").equals(2)).count()); + assertEquals(1, queryOutput.getRecords().stream().filter(r -> r.getValueInteger("orderId").equals(3)).count()); + assertEquals(2, queryOutput.getRecords().stream().filter(r -> r.getValueInteger("orderId").equals(4)).count()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOmsQueryByPersons() throws Exception + { + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + ///////////////////////////////////////////////////// + // inner join on bill-to person should find 6 rows // + ///////////////////////////////////////////////////// + queryInput.withQueryJoins(List.of(new QueryJoin(TestUtils.TABLE_NAME_PERSON).withJoinMetaData(instance.getJoin("orderJoinBillToPerson")).withSelect(true))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(6, queryOutput.getRecords().size(), "# of rows found by query"); + + ///////////////////////////////////////////////////// + // inner join on ship-to person should find 7 rows // + ///////////////////////////////////////////////////// + queryInput.withQueryJoins(List.of(new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withSelect(true))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(7, queryOutput.getRecords().size(), "# of rows found by query"); + + ///////////////////////////////////////////////////////////////////////////// + // inner join on both bill-to person and ship-to person should find 5 rows // + ///////////////////////////////////////////////////////////////////////////// + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true) + )); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "# of rows found by query"); + + ///////////////////////////////////////////////////////////////////////////// + // left join on both bill-to person and ship-to person should find 8 rows // + ///////////////////////////////////////////////////////////////////////////// + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withType(QueryJoin.Type.LEFT).withAlias("shipToPerson").withSelect(true), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withType(QueryJoin.Type.LEFT).withAlias("billToPerson").withSelect(true) + )); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(8, queryOutput.getRecords().size(), "# of rows found by query"); + + ////////////////////////////////////////////////// + // now join through to personalIdCard table too // + ////////////////////////////////////////////////// + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true), + new QueryJoin("billToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToIdCard").withSelect(true), + new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToIdCard").withSelect(true) + )); + queryInput.setFilter(new QQueryFilter() + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // look for billToPersons w/ idNumber starting with 1980 - should only be James and Darin (assert on that below). // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + .withCriteria(new QFilterCriteria("billToIdCard.idNumber", QCriteriaOperator.STARTS_WITH, "1980")) + ); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "# of rows found by query"); + assertThat(queryOutput.getRecords().stream().map(r -> r.getValueString("billToPerson.firstName")).toList()).allMatch(p -> p.equals("Darin") || p.equals("James")); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // ensure we throw if either of the ambiguous joins from person to id-card doesn't specify its left-table // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true), + new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToIdCard").withSelect(true), + new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToIdCard").withSelect(true) + )); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .rootCause() + .hasMessageContaining("Could not find a join between tables [order][personalIdCard]"); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // ensure we throw if either of the ambiguous joins from person to id-card doesn't specify its left-table // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true), + new QueryJoin("billToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToIdCard").withSelect(true), + new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToIdCard").withSelect(true) + )); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .rootCause() + .hasMessageContaining("Could not find a join between tables [order][personalIdCard]"); + + //////////////////////////////////////////////////////////////////////// + // ensure we throw if we have a bogus alias name given as a left-side // + //////////////////////////////////////////////////////////////////////// + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true), + new QueryJoin("notATable", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToIdCard").withSelect(true), + new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToIdCard").withSelect(true) + )); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .hasRootCauseMessage("Could not find a join between tables [notATable][personalIdCard]"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOmsQueryByPersonsExtraKelkhoffOrder() throws Exception + { + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert a second person w/ last name Kelkhoff, then an order for Darin Kelkhoff and this new Kelkhoff - // + // then query for orders w/ bill to person & ship to person both lastname = Kelkhoff, but different ids. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + Integer specialOrderId = 1701; + runTestSql("INSERT INTO person (id, first_name, last_name, email) VALUES (6, 'Jimmy', 'Kelkhoff', 'dk@gmail.com')", null); + runTestSql("INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (" + specialOrderId + ", 1, 1, 6)", null); + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withType(QueryJoin.Type.LEFT).withAlias("shipToPerson").withSelect(true), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withType(QueryJoin.Type.LEFT).withAlias("billToPerson").withSelect(true) + )); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("shipToPerson.lastName").withOperator(QCriteriaOperator.EQUALS).withOtherFieldName("billToPerson.lastName")) + .withCriteria(new QFilterCriteria().withFieldName("shipToPerson.id").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("billToPerson.id")) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); + assertEquals(specialOrderId, queryOutput.getRecords().get(0).getValueInteger("id")); + + //////////////////////////////////////////////////////////// + // re-run that query using personIds from the order table // + //////////////////////////////////////////////////////////// + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("shipToPerson.lastName").withOperator(QCriteriaOperator.EQUALS).withOtherFieldName("billToPerson.lastName")) + .withCriteria(new QFilterCriteria().withFieldName("order.shipToPersonId").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("order.billToPersonId")) + ); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); + assertEquals(specialOrderId, queryOutput.getRecords().get(0).getValueInteger("id")); + + /////////////////////////////////////////////////////////////////////////////////////////////// + // re-run that query using personIds from the order table, but not specifying the table name // + /////////////////////////////////////////////////////////////////////////////////////////////// + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("shipToPerson.lastName").withOperator(QCriteriaOperator.EQUALS).withOtherFieldName("billToPerson.lastName")) + .withCriteria(new QFilterCriteria().withFieldName("shipToPersonId").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("billToPersonId")) + ); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); + assertEquals(specialOrderId, queryOutput.getRecords().get(0).getValueInteger("id")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDuplicateAliases() + { + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson"), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson"), + new QueryJoin("billToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true), + new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true) // w/o alias, should get exception here - dupe table. + )); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .hasRootCauseMessage("Duplicate table name or alias: personalIdCard"); + + queryInput.withQueryJoins(List.of( + new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson"), + new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson"), + new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToPerson").withSelect(true), // dupe alias, should get exception here + new QueryJoin("billToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToPerson").withSelect(true) + )); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .hasRootCauseMessage("Duplicate table name or alias: shipToPerson"); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, also selecting item. + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoin() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("id") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on item to order + ** do a query on item, also selecting order. + ** This is a reverse of the above, to make sure join flipping, etc, is good. + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinReversed() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ITEM); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ORDER).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("description") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER + ".id") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, also selecting item, and also selecting orderLine... + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinAlsoSelectingInBetweenTable() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.withQueryJoins(List.of( + new QueryJoin(TestUtils.TABLE_NAME_ORDER_LINE).withType(QueryJoin.Type.INNER).withSelect(true), + new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true) + )); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(11); // one per line item + assertThat(records).allMatch(r -> r.getValue("id") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity") != null); + assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, filtered by item + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinWhereClauseOnly() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart"))); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(4); + assertThat(records).allMatch(r -> r.getValue("id") != null); + } + + + + /******************************************************************************* + ** Given tables: + ** order - orderLine - item + ** with exposedJoin on order to item + ** do a query on order, filtered by item + *******************************************************************************/ + @Test + void testTwoTableAwayExposedJoinWhereClauseBothJoinTables() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + QInstance instance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart")) + .withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity", QCriteriaOperator.IS_NOT_BLANK)) + ); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + List records = queryOutput.getRecords(); + assertThat(records).hasSize(4); + assertThat(records).allMatch(r -> r.getValue("id") != null); + } + + + + /******************************************************************************* + ** queries on the store table, where the primary key (id) is the security field + *******************************************************************************/ + @Test + void testRecordSecurityPrimaryKeyFieldNoFilters() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_STORE); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(3); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .anyMatch(r -> r.getValueInteger("id").equals(1)); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .anyMatch(r -> r.getValueInteger("id").equals(2)); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, null)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(Collections.emptyMap())); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .anyMatch(r -> r.getValueInteger("id").equals(1)) + .anyMatch(r -> r.getValueInteger("id").equals(3)); + } + + + + /******************************************************************************* + ** not really expected to be any different from where we filter on the primary key, + ** but just good to make sure + *******************************************************************************/ + @Test + void testRecordSecurityForeignKeyFieldNoFilters() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(8); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(3) + .allMatch(r -> r.getValueInteger("storeId").equals(1)); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .allMatch(r -> r.getValueInteger("storeId").equals(2)); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(Collections.emptyMap())); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(Map.of(TestUtils.TABLE_NAME_STORE, Collections.emptyList()))); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(6) + .allMatch(r -> r.getValueInteger("storeId").equals(1) || r.getValueInteger("storeId").equals(3)); + } + + + + /******************************************************************************* + ** Error seen in CTLive - query for order join lineItem, where lineItem's security + ** key is in order. + ** + ** Note - in this test-db setup, there happens to be a storeId in both order & + ** orderLine tables, so we can't quite reproduce the error we saw in CTL - so + ** query on different tables with the structure that'll produce the error. + *******************************************************************************/ + @Test + void testRequestedJoinWithTableWhoseSecurityFieldIsInMainTable() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_WAREHOUSE).withSelect(true)); + + ////////////////////////////////////////////// + // with the all-access key, find all 3 rows // + ////////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(3); + + /////////////////////////////////////////// + // with 1 security key value, find 1 row // + /////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .allMatch(r -> r.getValueInteger("storeId").equals(1)); + + /////////////////////////////////////////// + // with 1 security key value, find 1 row // + /////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .allMatch(r -> r.getValueInteger("storeId").equals(2)); + + ////////////////////////////////////////////////////////// + // with a mis-matching security key value, 0 rows found // + ////////////////////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + /////////////////////////////////////////////// + // with no security key values, 0 rows found // + /////////////////////////////////////////////// + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + //////////////////////////////////////////////// + // with null security key value, 0 rows found // + //////////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValues(Collections.emptyMap())); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + ////////////////////////////////////////////////////// + // with empty-list security key value, 0 rows found // + ////////////////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValues(Map.of(TestUtils.TABLE_NAME_STORE, Collections.emptyList()))); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + //////////////////////////////// + // with 2 values, find 2 rows // + //////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .allMatch(r -> r.getValueInteger("storeId").equals(1) || r.getValueInteger("storeId").equals(3)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithFilters() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(6); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .allMatch(r -> r.getValueInteger("storeId").equals(1)); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("storeId", QCriteriaOperator.IN, List.of(1, 2)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(3) + .allMatch(r -> r.getValueInteger("storeId").equals(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityFromJoinTableAlsoImplicitlyInQuery() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER_LINE); + + /////////////////////////////////////////////////////////////////////////////////////// + // orders 1, 2, and 3 are from store 1, so their lines (5 in total) should be found. // + // note, order 2 has the line with mis-matched store id - but, that shouldn't apply // + // here, because the line table's security comes from the order table. // + /////////////////////////////////////////////////////////////////////////////////////// + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("order.id", QCriteriaOperator.IN, List.of(1, 2, 3, 4)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(5); + + /////////////////////////////////////////////////////////////////// + // order 4 should be the only one found this time (with 2 lines) // + /////////////////////////////////////////////////////////////////// + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("order.id", QCriteriaOperator.IN, List.of(1, 2, 3, 4)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(2); + + //////////////////////////////////////////////////////////////// + // make sure we're also good if we explicitly join this table // + //////////////////////////////////////////////////////////////// + queryInput.withQueryJoin(new QueryJoin().withJoinTable(TestUtils.TABLE_NAME_ORDER).withSelect(true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(2); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithLockFromJoinTable() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // remove the normal lock on the order table - replace it with one from the joined store table // + ///////////////////////////////////////////////////////////////////////////////////////////////// + qInstance.getTable(TestUtils.TABLE_NAME_ORDER).getRecordSecurityLocks().clear(); + qInstance.getTable(TestUtils.TABLE_NAME_ORDER).withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TestUtils.TABLE_NAME_STORE) + .withJoinNameChain(List.of("orderJoinStore")) + .withFieldName("store.id")); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(6); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .allMatch(r -> r.getValueInteger("storeId").equals(1)); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("storeId", QCriteriaOperator.IN, List.of(1, 2)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(3) + .allMatch(r -> r.getValueInteger("storeId").equals(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE); + + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + Integer noOfOrders = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER)).getCount(); + Integer noOfOrderInstructions = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)).getCount(); + + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure we can join on order.current_order_instruction_id = order_instruction.id -- and that we get back 1 row per order // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderJoinCurrentOrderInstructions"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(noOfOrders, queryOutput.getRecords().size()); + } + + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // assert that the query succeeds (based on exposed join) if the joinMetaData isn't specified // + //////////////////////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(noOfOrders, queryOutput.getRecords().size()); + } + + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure we can join on order.id = order_instruction.order_id (e.g., not the exposed one used above) -- and that we get back 1 row per order instruction // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderInstructionsJoinOrder"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(noOfOrderInstructions, queryOutput.getRecords().size()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSecurityJoinForJoinedTableFromImplicitlyJoinedTable() throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////// + // in this test: // + // query on Order, joined with OrderLine. // + // Order has its own security field (storeId), that's always worked fine. // + // We want to change OrderLine's security field to be item.storeId - not order.storeId // + // so that item has to be brought into the query to secure the items. // + // this was originally broken, as it would generate a WHERE clause for item.storeId, // + // but it wouldn't put item in the FROM cluase. + ///////////////////////////////////////////////////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER_LINE) + .setRecordSecurityLocks(ListBuilder.of( + new RecordSecurityLock() + .withSecurityKeyType(TestUtils.TABLE_NAME_STORE) + .withFieldName("item.storeId") + .withJoinNameChain(List.of("orderLineJoinItem")))); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_LINE).withSelect(true)); + queryInput.withFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_ORDER_LINE + ".sku", QCriteriaOperator.IS_NOT_BLANK))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + List records = queryOutput.getRecords(); + assertEquals(3, records.size(), "expected no of records"); + + /////////////////////////////////////////////////////////////////////// + // we should get the orderLines for orders 4 and 5 - but not the one // + // for order 2, as it has an item from a different store // + /////////////////////////////////////////////////////////////////////// + assertThat(records).allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)); + } + + + + /******************************************************************************* + ** Addressing a regression where a table was brought into a query for its + ** security field, but it was a write-scope lock, so, it shouldn't have been. + *******************************************************************************/ + @Test + void testWriteLockOnJoinTableDoesntLimitQuery() throws Exception + { + /////////////////////////////////////////////////////////////////////// + // add a security key type for "idNumber" // + // then set up the person table with a read-write lock on that field // + /////////////////////////////////////////////////////////////////////// + QContext.getQInstance().addSecurityKeyType(new QSecurityKeyType().withName("idNumber")); + QTableMetaData personTable = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON); + personTable.withRecordSecurityLock(new RecordSecurityLock() + .withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE) + .withSecurityKeyType("idNumber") + .withFieldName(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber") + .withJoinNameChain(List.of(QJoinMetaData.makeInferredJoinName(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD)))); + + ///////////////////////////////////////////////////////////////////////////////////////// + // first, with no idNumber security key in session, query on person should find 0 rows // + ///////////////////////////////////////////////////////////////////////////////////////// + assertEquals(0, new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON)).getRecords().size()); + + /////////////////////////////////////////////////////////////////// + // put an idNumber in the session - query and find just that one // + /////////////////////////////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue("idNumber", "19800531")); + assertEquals(1, new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON)).getRecords().size()); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // change the lock to be scope=WRITE - and now, we should be able to see all of the records // + ////////////////////////////////////////////////////////////////////////////////////////////// + personTable.getRecordSecurityLocks().get(0).setLockScope(RecordSecurityLock.LockScope.WRITE); + assertEquals(5, new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON)).getRecords().size()); + } + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java index 223fa5f4..fa27d4ea 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java @@ -25,26 +25,20 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.io.Serializable; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; -import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now; import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset; @@ -52,14 +46,12 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -107,6 +99,34 @@ public class RDBMSQueryActionTest extends RDBMSActionTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testTrueQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.TRUE))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "'TRUE' query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFalseQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.FALSE))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "'FALSE' query should find no rows"); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -783,675 +803,6 @@ public class RDBMSQueryActionTest extends RDBMSActionTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testFilterFromJoinTableImplicitly() throws QException - { - QueryInput queryInput = initQueryRequest(); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("personalIdCard.idNumber", QCriteriaOperator.EQUALS, "19800531"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(1, queryOutput.getRecords().size(), "Query should find 1 rows"); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOneToOneInnerJoinWithoutWhere() throws QException - { - QueryInput queryInput = initQueryRequest(); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true)); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(3, queryOutput.getRecords().size(), "Join query should find 3 rows"); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("personalIdCard.idNumber").equals("19800531")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("personalIdCard.idNumber").equals("19800515")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("personalIdCard.idNumber").equals("19760528")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOneToOneLeftJoinWithoutWhere() throws QException - { - QueryInput queryInput = initQueryRequest(); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withType(QueryJoin.Type.LEFT).withSelect(true)); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(5, queryOutput.getRecords().size(), "Left Join query should find 5 rows"); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("personalIdCard.idNumber").equals("19800531")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("personalIdCard.idNumber").equals("19800515")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("personalIdCard.idNumber").equals("19760528")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Garret") && r.getValue("personalIdCard.idNumber") == null); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tyler") && r.getValue("personalIdCard.idNumber") == null); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOneToOneRightJoinWithoutWhere() throws QException - { - QueryInput queryInput = initQueryRequest(); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withType(QueryJoin.Type.RIGHT).withSelect(true)); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(6, queryOutput.getRecords().size(), "Right Join query should find 6 rows"); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("personalIdCard.idNumber").equals("19800531")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("personalIdCard.idNumber").equals("19800515")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("personalIdCard.idNumber").equals("19760528")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValue("firstName") == null && r.getValueString("personalIdCard.idNumber").equals("123123123")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValue("firstName") == null && r.getValueString("personalIdCard.idNumber").equals("987987987")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValue("firstName") == null && r.getValueString("personalIdCard.idNumber").equals("456456456")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOneToOneInnerJoinWithWhere() throws QException - { - QueryInput queryInput = initQueryRequest(); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true)); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber", QCriteriaOperator.STARTS_WITH, "1980"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(2, queryOutput.getRecords().size(), "Join query should find 2 rows"); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("personalIdCard.idNumber").equals("19800531")); - assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("personalIdCard.idNumber").equals("19800515")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOneToOneInnerJoinWithOrderBy() throws QException - { - QInstance qInstance = TestUtils.defineInstance(); - QueryInput queryInput = initQueryRequest(); - queryInput.withQueryJoin(new QueryJoin(qInstance.getJoin(TestUtils.TABLE_NAME_PERSON + "Join" + StringUtils.ucFirst(TestUtils.TABLE_NAME_PERSONAL_ID_CARD))).withSelect(true)); - queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(3, queryOutput.getRecords().size(), "Join query should find 3 rows"); - List idNumberListFromQuery = queryOutput.getRecords().stream().map(r -> r.getValueString(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber")).toList(); - assertEquals(List.of("19760528", "19800515", "19800531"), idNumberListFromQuery); - - ///////////////////////// - // repeat, sorted desc // - ///////////////////////// - queryInput.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber", false))); - queryOutput = new QueryAction().execute(queryInput); - assertEquals(3, queryOutput.getRecords().size(), "Join query should find 3 rows"); - idNumberListFromQuery = queryOutput.getRecords().stream().map(r -> r.getValueString(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber")).toList(); - assertEquals(List.of("19800531", "19800515", "19760528"), idNumberListFromQuery); - } - - - - /******************************************************************************* - ** In the prime data, we've got 1 order line set up with an item from a different - ** store than its order. Write a query to find such a case. - *******************************************************************************/ - @Test - void testFiveTableOmsJoinFindMismatchedStoreId() throws Exception - { - QueryInput queryInput = new QueryInput(); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER, TestUtils.TABLE_NAME_STORE).withAlias("orderStore").withSelect(true)); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER, TestUtils.TABLE_NAME_ORDER_LINE).withSelect(true)); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_LINE, TestUtils.TABLE_NAME_ITEM).withSelect(true)); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ITEM, TestUtils.TABLE_NAME_STORE).withAlias("itemStore").withSelect(true)); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria().withFieldName("orderStore.id").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("item.storeId"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); - QRecord qRecord = queryOutput.getRecords().get(0); - assertEquals(2, qRecord.getValueInteger("id")); - assertEquals(1, qRecord.getValueInteger("orderStore.id")); - assertEquals(2, qRecord.getValueInteger("itemStore.id")); - - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - // run the same setup, but this time, use the other-field-name as itemStore.id, instead of item.storeId // - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - queryInput.setFilter(new QQueryFilter(new QFilterCriteria().withFieldName("orderStore.id").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("itemStore.id"))); - queryOutput = new QueryAction().execute(queryInput); - assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); - qRecord = queryOutput.getRecords().get(0); - assertEquals(2, qRecord.getValueInteger("id")); - assertEquals(1, qRecord.getValueInteger("orderStore.id")); - assertEquals(2, qRecord.getValueInteger("itemStore.id")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOmsQueryByOrderLines() throws Exception - { - AtomicInteger orderLineCount = new AtomicInteger(); - runTestSql("SELECT COUNT(*) from order_line", (rs) -> - { - rs.next(); - orderLineCount.set(rs.getInt(1)); - }); - - QueryInput queryInput = new QueryInput(); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER_LINE); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER).withSelect(true)); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(orderLineCount.get(), queryOutput.getRecords().size(), "# of rows found by query"); - assertEquals(3, queryOutput.getRecords().stream().filter(r -> r.getValueInteger("order.id").equals(1)).count()); - assertEquals(1, queryOutput.getRecords().stream().filter(r -> r.getValueInteger("order.id").equals(2)).count()); - assertEquals(1, queryOutput.getRecords().stream().filter(r -> r.getValueInteger("orderId").equals(3)).count()); - assertEquals(2, queryOutput.getRecords().stream().filter(r -> r.getValueInteger("orderId").equals(4)).count()); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOmsQueryByPersons() throws Exception - { - QInstance instance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - - ///////////////////////////////////////////////////// - // inner join on bill-to person should find 6 rows // - ///////////////////////////////////////////////////// - queryInput.withQueryJoins(List.of(new QueryJoin(TestUtils.TABLE_NAME_PERSON).withJoinMetaData(instance.getJoin("orderJoinBillToPerson")).withSelect(true))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(6, queryOutput.getRecords().size(), "# of rows found by query"); - - ///////////////////////////////////////////////////// - // inner join on ship-to person should find 7 rows // - ///////////////////////////////////////////////////// - queryInput.withQueryJoins(List.of(new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withSelect(true))); - queryOutput = new QueryAction().execute(queryInput); - assertEquals(7, queryOutput.getRecords().size(), "# of rows found by query"); - - ///////////////////////////////////////////////////////////////////////////// - // inner join on both bill-to person and ship-to person should find 5 rows // - ///////////////////////////////////////////////////////////////////////////// - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true) - )); - queryOutput = new QueryAction().execute(queryInput); - assertEquals(5, queryOutput.getRecords().size(), "# of rows found by query"); - - ///////////////////////////////////////////////////////////////////////////// - // left join on both bill-to person and ship-to person should find 8 rows // - ///////////////////////////////////////////////////////////////////////////// - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withType(QueryJoin.Type.LEFT).withAlias("shipToPerson").withSelect(true), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withType(QueryJoin.Type.LEFT).withAlias("billToPerson").withSelect(true) - )); - queryOutput = new QueryAction().execute(queryInput); - assertEquals(8, queryOutput.getRecords().size(), "# of rows found by query"); - - ////////////////////////////////////////////////// - // now join through to personalIdCard table too // - ////////////////////////////////////////////////// - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true), - new QueryJoin("billToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToIdCard").withSelect(true), - new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToIdCard").withSelect(true) - )); - queryInput.setFilter(new QQueryFilter() - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // look for billToPersons w/ idNumber starting with 1980 - should only be James and Darin (assert on that below). // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - .withCriteria(new QFilterCriteria("billToIdCard.idNumber", QCriteriaOperator.STARTS_WITH, "1980")) - ); - queryOutput = new QueryAction().execute(queryInput); - assertEquals(3, queryOutput.getRecords().size(), "# of rows found by query"); - assertThat(queryOutput.getRecords().stream().map(r -> r.getValueString("billToPerson.firstName")).toList()).allMatch(p -> p.equals("Darin") || p.equals("James")); - - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // ensure we throw if either of the ambiguous joins from person to id-card doesn't specify its left-table // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true), - new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToIdCard").withSelect(true), - new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToIdCard").withSelect(true) - )); - assertThatThrownBy(() -> new QueryAction().execute(queryInput)) - .rootCause() - .hasMessageContaining("Could not find a join between tables [order][personalIdCard]"); - - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // ensure we throw if either of the ambiguous joins from person to id-card doesn't specify its left-table // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true), - new QueryJoin("billToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToIdCard").withSelect(true), - new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToIdCard").withSelect(true) - )); - assertThatThrownBy(() -> new QueryAction().execute(queryInput)) - .rootCause() - .hasMessageContaining("Could not find a join between tables [order][personalIdCard]"); - - //////////////////////////////////////////////////////////////////////// - // ensure we throw if we have a bogus alias name given as a left-side // - //////////////////////////////////////////////////////////////////////// - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson").withSelect(true), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson").withSelect(true), - new QueryJoin("notATable", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToIdCard").withSelect(true), - new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToIdCard").withSelect(true) - )); - assertThatThrownBy(() -> new QueryAction().execute(queryInput)) - .hasRootCauseMessage("Could not find a join between tables [notATable][personalIdCard]"); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOmsQueryByPersonsExtraKelkhoffOrder() throws Exception - { - QInstance instance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // insert a second person w/ last name Kelkhoff, then an order for Darin Kelkhoff and this new Kelkhoff - // - // then query for orders w/ bill to person & ship to person both lastname = Kelkhoff, but different ids. // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - Integer specialOrderId = 1701; - runTestSql("INSERT INTO person (id, first_name, last_name, email) VALUES (6, 'Jimmy', 'Kelkhoff', 'dk@gmail.com')", null); - runTestSql("INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (" + specialOrderId + ", 1, 1, 6)", null); - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withType(QueryJoin.Type.LEFT).withAlias("shipToPerson").withSelect(true), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withType(QueryJoin.Type.LEFT).withAlias("billToPerson").withSelect(true) - )); - queryInput.setFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria().withFieldName("shipToPerson.lastName").withOperator(QCriteriaOperator.EQUALS).withOtherFieldName("billToPerson.lastName")) - .withCriteria(new QFilterCriteria().withFieldName("shipToPerson.id").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("billToPerson.id")) - ); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); - assertEquals(specialOrderId, queryOutput.getRecords().get(0).getValueInteger("id")); - - //////////////////////////////////////////////////////////// - // re-run that query using personIds from the order table // - //////////////////////////////////////////////////////////// - queryInput.setFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria().withFieldName("shipToPerson.lastName").withOperator(QCriteriaOperator.EQUALS).withOtherFieldName("billToPerson.lastName")) - .withCriteria(new QFilterCriteria().withFieldName("order.shipToPersonId").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("order.billToPersonId")) - ); - queryOutput = new QueryAction().execute(queryInput); - assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); - assertEquals(specialOrderId, queryOutput.getRecords().get(0).getValueInteger("id")); - - /////////////////////////////////////////////////////////////////////////////////////////////// - // re-run that query using personIds from the order table, but not specifying the table name // - /////////////////////////////////////////////////////////////////////////////////////////////// - queryInput.setFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria().withFieldName("shipToPerson.lastName").withOperator(QCriteriaOperator.EQUALS).withOtherFieldName("billToPerson.lastName")) - .withCriteria(new QFilterCriteria().withFieldName("shipToPersonId").withOperator(QCriteriaOperator.NOT_EQUALS).withOtherFieldName("billToPersonId")) - ); - queryOutput = new QueryAction().execute(queryInput); - assertEquals(1, queryOutput.getRecords().size(), "# of rows found by query"); - assertEquals(specialOrderId, queryOutput.getRecords().get(0).getValueInteger("id")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testDuplicateAliases() - { - QInstance instance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson"), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson"), - new QueryJoin("billToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true), - new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true) // w/o alias, should get exception here - dupe table. - )); - assertThatThrownBy(() -> new QueryAction().execute(queryInput)) - .hasRootCauseMessage("Duplicate table name or alias: personalIdCard"); - - queryInput.withQueryJoins(List.of( - new QueryJoin(instance.getJoin("orderJoinShipToPerson")).withAlias("shipToPerson"), - new QueryJoin(instance.getJoin("orderJoinBillToPerson")).withAlias("billToPerson"), - new QueryJoin("shipToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("shipToPerson").withSelect(true), // dupe alias, should get exception here - new QueryJoin("billToPerson", TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("billToPerson").withSelect(true) - )); - assertThatThrownBy(() -> new QueryAction().execute(queryInput)) - .hasRootCauseMessage("Duplicate table name or alias: shipToPerson"); - } - - - - /******************************************************************************* - ** Given tables: - ** order - orderLine - item - ** with exposedJoin on order to item - ** do a query on order, also selecting item. - *******************************************************************************/ - @Test - void testTwoTableAwayExposedJoin() throws QException - { - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - - QInstance instance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - - queryInput.withQueryJoins(List.of( - new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true) - )); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - List records = queryOutput.getRecords(); - assertThat(records).hasSize(11); // one per line item - assertThat(records).allMatch(r -> r.getValue("id") != null); - assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null); - } - - - - /******************************************************************************* - ** Given tables: - ** order - orderLine - item - ** with exposedJoin on item to order - ** do a query on item, also selecting order. - ** This is a reverse of the above, to make sure join flipping, etc, is good. - *******************************************************************************/ - @Test - void testTwoTableAwayExposedJoinReversed() throws QException - { - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - - QInstance instance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ITEM); - - queryInput.withQueryJoins(List.of( - new QueryJoin(TestUtils.TABLE_NAME_ORDER).withType(QueryJoin.Type.INNER).withSelect(true) - )); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - List records = queryOutput.getRecords(); - assertThat(records).hasSize(11); // one per line item - assertThat(records).allMatch(r -> r.getValue("description") != null); - assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER + ".id") != null); - } - - - - /******************************************************************************* - ** Given tables: - ** order - orderLine - item - ** with exposedJoin on order to item - ** do a query on order, also selecting item, and also selecting orderLine... - *******************************************************************************/ - @Test - void testTwoTableAwayExposedJoinAlsoSelectingInBetweenTable() throws QException - { - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - - QInstance instance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - - queryInput.withQueryJoins(List.of( - new QueryJoin(TestUtils.TABLE_NAME_ORDER_LINE).withType(QueryJoin.Type.INNER).withSelect(true), - new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true) - )); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - List records = queryOutput.getRecords(); - assertThat(records).hasSize(11); // one per line item - assertThat(records).allMatch(r -> r.getValue("id") != null); - assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity") != null); - assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null); - } - - - - /******************************************************************************* - ** Given tables: - ** order - orderLine - item - ** with exposedJoin on order to item - ** do a query on order, filtered by item - *******************************************************************************/ - @Test - void testTwoTableAwayExposedJoinWhereClauseOnly() throws QException - { - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - - QInstance instance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart"))); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - List records = queryOutput.getRecords(); - assertThat(records).hasSize(4); - assertThat(records).allMatch(r -> r.getValue("id") != null); - } - - - - /******************************************************************************* - ** Given tables: - ** order - orderLine - item - ** with exposedJoin on order to item - ** do a query on order, filtered by item - *******************************************************************************/ - @Test - void testTwoTableAwayExposedJoinWhereClauseBothJoinTables() throws QException - { - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - - QInstance instance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.setFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart")) - .withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity", QCriteriaOperator.IS_NOT_BLANK)) - ); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - List records = queryOutput.getRecords(); - assertThat(records).hasSize(4); - assertThat(records).allMatch(r -> r.getValue("id") != null); - } - - - - /******************************************************************************* - ** queries on the store table, where the primary key (id) is the security field - *******************************************************************************/ - @Test - void testRecordSecurityPrimaryKeyFieldNoFilters() throws QException - { - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_STORE); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(3); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(1) - .anyMatch(r -> r.getValueInteger("id").equals(1)); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(1) - .anyMatch(r -> r.getValueInteger("id").equals(2)); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - QContext.setQSession(new QSession()); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, null)); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - QContext.setQSession(new QSession().withSecurityKeyValues(Map.of(TestUtils.TABLE_NAME_STORE, Collections.emptyList()))); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(2) - .anyMatch(r -> r.getValueInteger("id").equals(1)) - .anyMatch(r -> r.getValueInteger("id").equals(3)); - } - - - - /******************************************************************************* - ** not really expected to be any different from where we filter on the primary key, - ** but just good to make sure - *******************************************************************************/ - @Test - void testRecordSecurityForeignKeyFieldNoFilters() throws QException - { - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(8); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(3) - .allMatch(r -> r.getValueInteger("storeId").equals(1)); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(2) - .allMatch(r -> r.getValueInteger("storeId").equals(2)); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - QContext.setQSession(new QSession()); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, null)); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - QContext.setQSession(new QSession().withSecurityKeyValues(Map.of(TestUtils.TABLE_NAME_STORE, Collections.emptyList()))); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(6) - .allMatch(r -> r.getValueInteger("storeId").equals(1) || r.getValueInteger("storeId").equals(3)); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testRecordSecurityWithFilters() throws QException - { - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(6); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(2) - .allMatch(r -> r.getValueInteger("storeId").equals(1)); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); - QContext.setQSession(new QSession()); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("storeId", QCriteriaOperator.IN, List.of(1, 2)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(3) - .allMatch(r -> r.getValueInteger("storeId").equals(1)); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testRecordSecurityFromJoinTableAlsoImplicitlyInQuery() throws QException - { - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER_LINE); - - /////////////////////////////////////////////////////////////////////////////////////////// - // orders 1, 2, and 3 are from store 1, so their lines (5 in total) should be found. // - // note, order 2 has the line with mis-matched store id - but, that shouldn't apply here // - /////////////////////////////////////////////////////////////////////////////////////////// - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("order.id", QCriteriaOperator.IN, List.of(1, 2, 3, 4)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); - assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(5); - - /////////////////////////////////////////////////////////////////// - // order 4 should be the only one found this time (with 2 lines) // - /////////////////////////////////////////////////////////////////// - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("order.id", QCriteriaOperator.IN, List.of(1, 2, 3, 4)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); - assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(2); - - //////////////////////////////////////////////////////////////// - // make sure we're also good if we explicitly join this table // - //////////////////////////////////////////////////////////////// - queryInput.withQueryJoin(new QueryJoin().withJoinTable(TestUtils.TABLE_NAME_ORDER).withSelect(true)); - assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(2); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -1627,68 +978,6 @@ public class RDBMSQueryActionTest extends RDBMSActionTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testRecordSecurityWithLockFromJoinTable() throws QException - { - QInstance qInstance = TestUtils.defineInstance(); - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - - ///////////////////////////////////////////////////////////////////////////////////////////////// - // remove the normal lock on the order table - replace it with one from the joined store table // - ///////////////////////////////////////////////////////////////////////////////////////////////// - qInstance.getTable(TestUtils.TABLE_NAME_ORDER).getRecordSecurityLocks().clear(); - qInstance.getTable(TestUtils.TABLE_NAME_ORDER).withRecordSecurityLock(new RecordSecurityLock() - .withSecurityKeyType(TestUtils.TABLE_NAME_STORE) - .withJoinNameChain(List.of("orderJoinStore")) - .withFieldName("store.id")); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(6); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(2) - .allMatch(r -> r.getValueInteger("storeId").equals(1)); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.BETWEEN, List.of(2, 7)))); - QContext.setQSession(new QSession()); - assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); - - queryInput.setFilter(new QQueryFilter(new QFilterCriteria("storeId", QCriteriaOperator.IN, List.of(1, 2)))); - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(3) - .allMatch(r -> r.getValueInteger("storeId").equals(1)); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException - { - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE); - - assertThat(new QueryAction().execute(queryInput).getRecords()) - .hasSize(1); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -1718,51 +1007,4 @@ public class RDBMSQueryActionTest extends RDBMSActionTest } - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException - { - QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); - - { - ///////////////////////////////////////////////////////// - // assert a failure if the join to use isn't specified // - ///////////////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)); - assertThatThrownBy(() -> new QueryAction().execute(queryInput)).rootCause().hasMessageContaining("More than 1 join was found"); - } - - Integer noOfOrders = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER)).getCount(); - Integer noOfOrderInstructions = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)).getCount(); - - { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // make sure we can join on order.current_order_instruction_id = order_instruction.id -- and that we get back 1 row per order // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderJoinCurrentOrderInstructions"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(noOfOrders, queryOutput.getRecords().size()); - } - - { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // make sure we can join on order.id = order_instruction.order_id -- and that we get back 1 row per order instruction // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); - queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderInstructionsJoinOrder"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(noOfOrderInstructions, queryOutput.getRecords().size()); - } - - } - } diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java new file mode 100644 index 00000000..c6b8fc17 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java @@ -0,0 +1,70 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for RDBMSTableMetaDataBuilder + *******************************************************************************/ +class RDBMSTableMetaDataBuilderTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + TestUtils.primeTestDatabase("prime-test-database.sql"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QBackendMetaData backend = QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME); + QTableMetaData table = new RDBMSTableMetaDataBuilder().buildTableMetaData((RDBMSBackendMetaData) backend, "order"); + + assertNotNull(table); + assertEquals("order", table.getName()); + assertEquals("id", table.getPrimaryKeyField()); + assertEquals(QFieldType.INTEGER, table.getField(table.getPrimaryKeyField()).getType()); + assertNotNull(table.getField("storeId")); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java index e49bc014..9e3be3ed 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java @@ -23,25 +23,52 @@ package com.kingsrook.qqq.backend.module.rdbms.reporting; import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryStorageAction; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -196,6 +223,220 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest + /******************************************************************************* + ** + *******************************************************************************/ + private RunProcessOutput runSavedReport(SavedReport savedReport, ReportFormatPossibleValueEnum reportFormat) throws Exception + { + savedReport.setLabel("Test Report"); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + if(QContext.getQInstance().getTable(SavedReport.TABLE_NAME) == null) + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + } + + QRecord savedReportRecord = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords().get(0); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(RenderSavedReportMetaDataProducer.NAME); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setCallback(QProcessCallbackFactory.forRecord(savedReportRecord)); + input.addValue("reportFormat", reportFormat); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + return (runProcessOutput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List runSavedReportForCSV(SavedReport savedReport) throws Exception + { + RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.CSV); + + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference)); + + return (IOUtils.readLines(inputStream, StandardCharsets.UTF_8)); + } + + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinSelections() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId") + .withColumn("orderInstructions.instructions"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()))); + + assertEquals(""" + "Id","Store","Order Instructions: Instructions" + """.trim(), lines.get(0)); + assertEquals(""" + "1","Q-Mart","order 1 v2" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinSelectedAndOrdered() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId") + .withColumn("orderInstructions.instructions"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withOrderBy(new QFilterOrderBy("orderInstructions.id", false)) + ))); + + assertEquals(""" + "Id","Store","Order Instructions: Instructions" + """.trim(), lines.get(0)); + assertEquals(""" + "8","QDepot","order 8 v1" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinCriteria() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("orderInstructions.instructions", QCriteriaOperator.CONTAINS, "v3")) + ))); + + assertEquals(""" + "Id","Store" + """.trim(), lines.get(0)); + assertEquals(""" + "2","Q-Mart" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithExposedJoinMultipleTablesAwaySelected() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId") + .withColumn("item.description"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()))); + + assertEquals(""" + "Id","Store","Item: Description" + """.trim(), lines.get(0)); + assertEquals(""" + "1","Q-Mart","Q-Mart Item 1" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithExposedJoinMultipleTablesAwayAsCriteria() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("item.description", QCriteriaOperator.CONTAINS, "Item 7")) + ))); + + assertEquals(""" + "Id","Store" + """.trim(), lines.get(0)); + assertEquals(""" + "6","QDepot" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithPivotsFromJoinTable() throws Exception + { + SavedReport savedReport = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("item.storeId") + .withColumn("item.description"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("item.storeId")) + .withValue(new PivotTableValue().withFieldName("item.description").withFunction(PivotTableFunction.COUNT)))); + + ////////////////////////////////////////////// + // make sure we can render xlsx w/o a crash // + ////////////////////////////////////////////// + RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.XLSX); + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference)); + + String path = "/tmp/pivot.xlsx"; + inputStream.transferTo(new FileOutputStream(path)); + // LocalMacDevUtils.mayOpenFiles = true; + LocalMacDevUtils.openFile(path); + + /////////////////////////////////////////////////////// + // render as csv too - and assert about those values // + /////////////////////////////////////////////////////// + List csv = runSavedReportForCSV(savedReport); + System.out.println(StringUtils.join("\n", csv)); + assertEquals(""" + "Store","Count Of Item: Description\"""", csv.get(0)); + assertEquals(""" + "Q-Mart","4\"""", csv.get(1)); + assertEquals(""" + "Totals","11\"""", csv.get(4)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -204,9 +445,12 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest ReportInput reportInput = new ReportInput(); QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); reportInput.setReportName(TEST_REPORT); - reportInput.setReportFormat(ReportFormat.CSV); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - reportInput.setReportOutputStream(outputStream); + + reportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.CSV) + .withReportOutputStream(outputStream)); + new GenerateReportAction().execute(reportInput); return (outputStream.toString()); } diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/SharingMetaDataProvider.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/SharingMetaDataProvider.java new file mode 100644 index 00000000..9dccf88d --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/SharingMetaDataProvider.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.sharing; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.Asset; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.Client; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.Group; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.SharedAsset; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.User; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SharingMetaDataProvider +{ + public static final String USER_ID_KEY_TYPE = "userIdKey"; + public static final String USER_ID_ALL_ACCESS_KEY_TYPE = "userIdAllAccessKey"; + + public static final String GROUP_ID_KEY_TYPE = "groupIdKey"; + public static final String GROUP_ID_ALL_ACCESS_KEY_TYPE = "groupIdAllAccessKey"; + + private static final String ASSET_JOIN_SHARED_ASSET = "assetJoinSharedAsset"; + private static final String SHARED_ASSET_JOIN_ASSET = "sharedAssetJoinAsset"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void defineAll(QInstance qInstance) throws QException + { + qInstance.addSecurityKeyType(new QSecurityKeyType() + .withName(USER_ID_KEY_TYPE) + .withAllAccessKeyName(USER_ID_ALL_ACCESS_KEY_TYPE)); + + qInstance.addSecurityKeyType(new QSecurityKeyType() + .withName(GROUP_ID_KEY_TYPE) + .withAllAccessKeyName(GROUP_ID_ALL_ACCESS_KEY_TYPE)); + + qInstance.addTable(new QTableMetaData() + .withName(Asset.TABLE_NAME) + .withPrimaryKeyField("id") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withBackendDetails(new RDBMSTableBackendDetails().withTableName("asset")) + .withFieldsFromEntity(Asset.class) + + //////////////////////////////////////// + // This is original - just owner/user // + //////////////////////////////////////// + // .withRecordSecurityLock(new RecordSecurityLock() + // .withSecurityKeyType(USER_ID_KEY_TYPE) + // .withFieldName("userId"))); + + .withRecordSecurityLock(new MultiRecordSecurityLock() + .withOperator(MultiRecordSecurityLock.BooleanOperator.OR) + .withLock(new RecordSecurityLock() + .withSecurityKeyType(USER_ID_KEY_TYPE) + .withFieldName("userId")) + .withLock(new RecordSecurityLock() + .withSecurityKeyType(USER_ID_KEY_TYPE) + .withFieldName(SharedAsset.TABLE_NAME + ".userId") + .withJoinNameChain(List.of(SHARED_ASSET_JOIN_ASSET))) + .withLock(new RecordSecurityLock() + .withSecurityKeyType(GROUP_ID_KEY_TYPE) + .withFieldName(SharedAsset.TABLE_NAME + ".groupId") + .withJoinNameChain(List.of(SHARED_ASSET_JOIN_ASSET))) + )); + QInstanceEnricher.setInferredFieldBackendNames(qInstance.getTable(Asset.TABLE_NAME)); + + qInstance.addTable(new QTableMetaData() + .withName(SharedAsset.TABLE_NAME) + .withBackendDetails(new RDBMSTableBackendDetails().withTableName("shared_asset")) + .withPrimaryKeyField("id") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withFieldsFromEntity(SharedAsset.class) + .withRecordSecurityLock(new MultiRecordSecurityLock() + .withOperator(MultiRecordSecurityLock.BooleanOperator.OR) + .withLock(new RecordSecurityLock() + .withSecurityKeyType(USER_ID_KEY_TYPE) + .withFieldName("userId")) + .withLock(new RecordSecurityLock() + .withSecurityKeyType(GROUP_ID_KEY_TYPE) + .withFieldName("groupId")) + )); + QInstanceEnricher.setInferredFieldBackendNames(qInstance.getTable(SharedAsset.TABLE_NAME)); + + qInstance.addTable(new QTableMetaData() + .withName(User.TABLE_NAME) + .withPrimaryKeyField("id") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withFieldsFromEntity(User.class) + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(USER_ID_KEY_TYPE) + .withFieldName("id"))); + QInstanceEnricher.setInferredFieldBackendNames(qInstance.getTable(User.TABLE_NAME)); + + qInstance.addTable(new QTableMetaData() + .withName(Group.TABLE_NAME) + .withPrimaryKeyField("id") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withFieldsFromEntity(Group.class)); + QInstanceEnricher.setInferredFieldBackendNames(qInstance.getTable(Group.TABLE_NAME)); + + qInstance.addTable(new QTableMetaData() + .withName(Client.TABLE_NAME) + .withPrimaryKeyField("id") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withFieldsFromEntity(Client.class)); + QInstanceEnricher.setInferredFieldBackendNames(qInstance.getTable(Client.TABLE_NAME)); + + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(User.TABLE_NAME)); + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(Group.TABLE_NAME)); + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(Client.TABLE_NAME)); + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(Asset.TABLE_NAME)); + + qInstance.addJoin(new QJoinMetaData() + .withName(ASSET_JOIN_SHARED_ASSET) + .withLeftTable(Asset.TABLE_NAME) + .withRightTable(SharedAsset.TABLE_NAME) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "assetId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName(SHARED_ASSET_JOIN_ASSET) + .withLeftTable(SharedAsset.TABLE_NAME) + .withRightTable(Asset.TABLE_NAME) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("assetId", "id")) + ); + } + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/SharingTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/SharingTest.java new file mode 100644 index 00000000..01dac2ec --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/SharingTest.java @@ -0,0 +1,516 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.sharing; + + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.Asset; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.Group; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.SharedAsset; +import com.kingsrook.qqq.backend.module.rdbms.sharing.model.User; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.module.rdbms.sharing.SharingMetaDataProvider.GROUP_ID_ALL_ACCESS_KEY_TYPE; +import static com.kingsrook.qqq.backend.module.rdbms.sharing.SharingMetaDataProvider.GROUP_ID_KEY_TYPE; +import static com.kingsrook.qqq.backend.module.rdbms.sharing.SharingMetaDataProvider.USER_ID_ALL_ACCESS_KEY_TYPE; +import static com.kingsrook.qqq.backend.module.rdbms.sharing.SharingMetaDataProvider.USER_ID_KEY_TYPE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SharingTest +{ + ////////////// + // user ids // + ////////////// + public static final int HOMER_ID = 1; + public static final int MARGE_ID = 2; + public static final int BART_ID = 3; + public static final int LISA_ID = 4; + public static final int BURNS_ID = 5; + + /////////////// + // group ids // + /////////////// + public static final int SIMPSONS_ID = 1; + public static final int POWER_PLANT_ID = 2; + public static final int BOGUS_GROUP_ID = Integer.MAX_VALUE; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + TestUtils.primeTestDatabase("prime-test-database-sharing-test.sql"); + + QInstance qInstance = TestUtils.defineInstance(); + SharingMetaDataProvider.defineAll(qInstance); + + QContext.init(qInstance, new QSession()); + + loadData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void loadData() throws QException + { + QContext.getQSession().withSecurityKeyValue(SharingMetaDataProvider.USER_ID_ALL_ACCESS_KEY_TYPE, true); + + List userList = List.of( + new User().withId(HOMER_ID).withUsername("homer"), + new User().withId(MARGE_ID).withUsername("marge"), + new User().withId(BART_ID).withUsername("bart"), + new User().withId(LISA_ID).withUsername("lisa"), + new User().withId(BURNS_ID).withUsername("burns")); + new InsertAction().execute(new InsertInput(User.TABLE_NAME).withRecordEntities(userList)); + + List groupList = List.of( + new Group().withId(SIMPSONS_ID).withName("simpsons"), + new Group().withId(POWER_PLANT_ID).withName("powerplant")); + new InsertAction().execute(new InsertInput(Group.TABLE_NAME).withRecordEntities(groupList)); + + List assetList = List.of( + new Asset().withId(1).withName("742evergreen").withUserId(HOMER_ID), + new Asset().withId(2).withName("beer").withUserId(HOMER_ID), + new Asset().withId(3).withName("car").withUserId(MARGE_ID), + new Asset().withId(4).withName("skateboard").withUserId(BART_ID), + new Asset().withId(5).withName("santaslittlehelper").withUserId(BART_ID), + new Asset().withId(6).withName("saxamaphone").withUserId(LISA_ID), + new Asset().withId(7).withName("radiation").withUserId(BURNS_ID)); + new InsertAction().execute(new InsertInput(Asset.TABLE_NAME).withRecordEntities(assetList)); + + List sharedAssetList = List.of( + new SharedAsset().withAssetId(1).withGroupId(SIMPSONS_ID), // homer shares his house with the simpson family (group) + new SharedAsset().withAssetId(3).withUserId(HOMER_ID), // marge shares a car with homer + new SharedAsset().withAssetId(5).withGroupId(SIMPSONS_ID), // bart shares santa's little helper with the whole family + new SharedAsset().withAssetId(7).withGroupId(POWER_PLANT_ID) // burns shares radiation with the power plant + ); + new InsertAction().execute(new InsertInput(SharedAsset.TABLE_NAME).withRecordEntities(sharedAssetList)); + + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueryAssetWithUserIdOnlySecurityKey() throws QException + { + //////////////////////////////////////////////////////////////////// + // update the asset table to change its lock to only be on userId // + //////////////////////////////////////////////////////////////////// + QContext.getQInstance().getTable(Asset.TABLE_NAME) + .withRecordSecurityLocks(List.of(new RecordSecurityLock() + .withSecurityKeyType(USER_ID_KEY_TYPE) + .withFieldName("userId"))); + + //////////////////////////////////////////////////////// + // with nothing in session, make sure we find nothing // + //////////////////////////////////////////////////////// + assertEquals(0, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + + //////////////////////////////////// + // marge direct owner only of car // + //////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, MARGE_ID); + assertEquals(1, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + + ///////////////////////////////////////////////// + // homer direct owner of 742evergreen and beer // + ///////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, HOMER_ID); + assertEquals(2, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + + ///////////////////////////////////////////////////// + // marge & homer - own car, 742evergreen, and beer // + ///////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, HOMER_ID); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, MARGE_ID); + assertEquals(3, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + } + + + + /******************************************************************************* + ** normally (?) maybe we wouldn't query sharedAsset directly (we'd instead query + ** for asset, and understand that there's a security lock coming from sharedAsset), + ** but this test is here as we build up making a more complex lock like that. + *******************************************************************************/ + @Test + void testQuerySharedAssetDirectly() throws QException + { + //////////////////////////////////////////////////////// + // with nothing in session, make sure we find nothing // + //////////////////////////////////////////////////////// + assertEquals(0, new QueryAction().execute(new QueryInput(SharedAsset.TABLE_NAME)).getRecords().size()); + + ///////////////////////////////////// + // homer has a car shared with him // + ///////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, HOMER_ID); + assertEquals(1, new QueryAction().execute(new QueryInput(SharedAsset.TABLE_NAME)).getRecords().size()); + + ///////////////////////////////////////////////////////////////////////////////////////// + // now put homer's groups in the session as well - and we should find 742evergreen too // + ///////////////////////////////////////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, HOMER_ID); + QContext.getQSession().withSecurityKeyValue(GROUP_ID_KEY_TYPE, SIMPSONS_ID); + QContext.getQSession().withSecurityKeyValue(GROUP_ID_KEY_TYPE, POWER_PLANT_ID); + List records = new QueryAction().execute(new QueryInput(SharedAsset.TABLE_NAME)).getRecords(); + assertEquals(4, records.size()); + } + + + + /******************************************************************************* + ** real-world use-case (e.g., why sharing concept exists) - query the asset table + ** + *******************************************************************************/ + @Test + void testQueryAssetsWithLockThroughSharing() throws QException, SQLException + { + //////////////////////////////////////////////////////// + // with nothing in session, make sure we find nothing // + //////////////////////////////////////////////////////// + assertEquals(0, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // homer has a car shared with him and 2 things he owns himself - so w/ only his userId in session (and no groups), should find those 3 // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, HOMER_ID); + assertEquals(3, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + + ////////////////////////////////////////////////////////////////////// + // add a group that matches nothing now, just to ensure same result // + ////////////////////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValue(GROUP_ID_KEY_TYPE, BOGUS_GROUP_ID); + assertEquals(3, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now put homer's groups in the session as well - and we should find the 3 from above, plus a shared family asset and shared power-plant asset // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, HOMER_ID); + QContext.getQSession().withSecurityKeyValue(GROUP_ID_KEY_TYPE, SIMPSONS_ID); + QContext.getQSession().withSecurityKeyValue(GROUP_ID_KEY_TYPE, POWER_PLANT_ID); + assertEquals(5, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueryAllAccessKeys() throws QException + { + /////////////////////////////////////////////////////////////// + // with user-id all access key, should get all asset records // + /////////////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_ALL_ACCESS_KEY_TYPE, true); + assertEquals(7, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // with group-id all access key... // + // the original thought was, that we should get all assets which are shared to any group // + // but the code that we first wrote generates SQL w/ an OR (1=1) clause, meaning we get all // + // assets, which makes some sense too, so we'll go with that for now... // + ////////////////////////////////////////////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(GROUP_ID_ALL_ACCESS_KEY_TYPE, true); + assertEquals(7, new QueryAction().execute(new QueryInput(Asset.TABLE_NAME)).getRecords().size()); + } + + + + /******************************************************************************* + ** if I'm only able to access user 1 and 2, I shouldn't be able to share to user 3 + *******************************************************************************/ + @Test + void testInsertUpdateDeleteShareUserIdKey() throws QException, SQLException + { + SharedAsset recordToInsert = new SharedAsset().withUserId(3).withAssetId(6); + + ///////////////////////////////////////// + // empty set of keys should give error // + ///////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(SharedAsset.TABLE_NAME).withRecordEntity(recordToInsert)); + assertThat(insertOutput.getRecords().get(0).getErrors()).isNotEmpty(); + + ///////////////////////////////////////////// + // mis-matched keys should give same error // + ///////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, 1); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, 2); + insertOutput = new InsertAction().execute(new InsertInput(SharedAsset.TABLE_NAME).withRecordEntity(recordToInsert)); + assertThat(insertOutput.getRecords().get(0).getErrors()).isNotEmpty(); + + ///////////////////////////////////////////////////////// + // then if I get user 3, I can insert the share for it // + ///////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, 3); + insertOutput = new InsertAction().execute(new InsertInput(SharedAsset.TABLE_NAME).withRecordEntity(recordToInsert)); + assertThat(insertOutput.getRecords().get(0).getErrors()).isEmpty(); + + ///////////////////////////////////////// + // get ready for a sequence of updates // + ///////////////////////////////////////// + Integer shareId = insertOutput.getRecords().get(0).getValueInteger("id"); + Supplier makeRecordToUpdate = () -> new QRecord().withValue("id", shareId).withValue("modifyDate", Instant.now()); + + /////////////////////////////////////////////////////////////////////////////// + // now w/o user 3 in my session, I shouldn't be allowed to update that share // + // start w/ empty security keys // + /////////////////////////////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(SharedAsset.TABLE_NAME).withRecord(makeRecordToUpdate.get())); + assertThat(updateOutput.getRecords().get(0).getErrors()) + .anyMatch(e -> e.getMessage().contains("No record was found")); // because w/o the key, you can't even see it. + + ///////////////////////////////////////////// + // mis-matched keys should give same error // + ///////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, 1); + updateOutput = new UpdateAction().execute(new UpdateInput(SharedAsset.TABLE_NAME).withRecord(makeRecordToUpdate.get())); + assertThat(updateOutput.getRecords().get(0).getErrors()) + .anyMatch(e -> e.getMessage().contains("No record was found")); // because w/o the key, you can't even see it. + + ////////////////////////////////////////////////// + // now with user id 3, should be able to update // + ////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, 3); + updateOutput = new UpdateAction().execute(new UpdateInput(SharedAsset.TABLE_NAME).withRecord(makeRecordToUpdate.get())); + assertThat(updateOutput.getRecords().get(0).getErrors()).isEmpty(); + + ////////////////////////////////////////////////////////////////////////// + // now see if you can update to a user that you don't have (you can't!) // + ////////////////////////////////////////////////////////////////////////// + /* todo - here's where the logic in ValidateRecordSecurityLockHelper fails us... + updateOutput = new UpdateAction().execute(new UpdateInput(SharedAsset.TABLE_NAME).withRecord(makeRecordToUpdate.get().withValue("userId", 2))); + assertThat(updateOutput.getRecords().get(0).getErrors()).isNotEmpty(); + */ + + /////////////////////////////////////////////////////////////////////// + // Add that user (2) to the session - then the update should succeed // + /////////////////////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, 2); + updateOutput = new UpdateAction().execute(new UpdateInput(SharedAsset.TABLE_NAME).withRecord(makeRecordToUpdate.get().withValue("userId", 2))); + assertThat(updateOutput.getRecords().get(0).getErrors()).isEmpty(); + + /////////////////////////////////////////////// + // now move on to deletes - first empty keys // + /////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(SharedAsset.TABLE_NAME).withPrimaryKey(shareId)); + assertEquals(0, deleteOutput.getDeletedRecordCount()); // can't even find it, so no error to be reported. + + /////////////////////// + // next mismatch key // + /////////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, 1); + deleteOutput = new DeleteAction().execute(new DeleteInput(SharedAsset.TABLE_NAME).withPrimaryKey(shareId)); + assertEquals(0, deleteOutput.getDeletedRecordCount()); // can't even find it, so no error to be reported. + + /////////////////// + // next success! // + /////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, 2); + deleteOutput = new DeleteAction().execute(new DeleteInput(SharedAsset.TABLE_NAME).withPrimaryKey(shareId)); + assertEquals(1, deleteOutput.getDeletedRecordCount()); + } + + + + /******************************************************************************* + ** useful to debug (e.g., to see inside h2). add calls as needed. + *******************************************************************************/ + private void printSQL(String sql) throws SQLException + { + Connection connection = new ConnectionManager().getConnection((RDBMSBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME)); + List> maps = QueryManager.executeStatementForRows(connection, sql); + System.out.println(sql); + maps.forEach(System.out::println); + } + + + + /******************************************************************************* + ** if I only have access to group 1, make sure I can't share to group 2 + *******************************************************************************/ + @Test + void testInsertShareGroupIdKey() throws QException + { + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(GROUP_ID_KEY_TYPE, 1); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(SharedAsset.TABLE_NAME).withRecordEntity(new SharedAsset().withGroupId(2).withAssetId(6))); + assertThat(insertOutput.getRecords().get(0).getErrors()).isNotEmpty(); + + ////////////////////////////////////////// + // add group 2, then we can share to it // + ////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValue(GROUP_ID_KEY_TYPE, 2); + insertOutput = new InsertAction().execute(new InsertInput(SharedAsset.TABLE_NAME).withRecordEntity(new SharedAsset().withGroupId(2).withAssetId(6))); + assertThat(insertOutput.getRecords().get(0).getErrors()).isEmpty(); + } + + + + /******************************************************************************* + ** w/ user-all-access key, can insert shares for any user + *******************************************************************************/ + @Test + void testInsertUpdateDeleteShareUserAllAccessKey() throws QException + { + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_ALL_ACCESS_KEY_TYPE, true); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(SharedAsset.TABLE_NAME).withRecordEntity(new SharedAsset().withUserId(1).withAssetId(4))); + assertThat(insertOutput.getRecords().get(0).getErrors()).isEmpty(); + + ///////////////////////////////////////// + // get ready for a sequence of updates // + ///////////////////////////////////////// + Integer shareId = insertOutput.getRecords().get(0).getValueInteger("id"); + Supplier makeRecordToUpdate = () -> new QRecord().withValue("id", shareId).withValue("modifyDate", Instant.now()); + + ////////////////////////////////// + // now w/o all-access key, fail // + ////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(SharedAsset.TABLE_NAME).withRecord(makeRecordToUpdate.get())); + assertThat(updateOutput.getRecords().get(0).getErrors()) + .anyMatch(e -> e.getMessage().contains("No record was found")); // because w/o the key, you can't even see it. + + /////////////////////////////////////////////////////// + // now with all-access key, should be able to update // + /////////////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_ALL_ACCESS_KEY_TYPE, true); + updateOutput = new UpdateAction().execute(new UpdateInput(SharedAsset.TABLE_NAME).withRecord(makeRecordToUpdate.get().withValue("userId", 2))); + assertThat(updateOutput.getRecords().get(0).getErrors()).isEmpty(); + + /////////////////////////////////////////////// + // now move on to deletes - first empty keys // + /////////////////////////////////////////////// + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(SharedAsset.TABLE_NAME).withPrimaryKey(shareId)); + assertEquals(0, deleteOutput.getDeletedRecordCount()); // can't even find it, so no error to be reported. + + /////////////////// + // next success! // + /////////////////// + QContext.getQSession().withSecurityKeyValue(USER_ID_ALL_ACCESS_KEY_TYPE, true); + deleteOutput = new DeleteAction().execute(new DeleteInput(SharedAsset.TABLE_NAME).withPrimaryKey(shareId)); + assertEquals(1, deleteOutput.getDeletedRecordCount()); + } + + + + /******************************************************************************* + ** w/ group-all-access key, can insert shares for any group + *******************************************************************************/ + @Test + void testInsertShareGroupAllAccessKey() throws QException + { + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(GROUP_ID_ALL_ACCESS_KEY_TYPE, true); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(SharedAsset.TABLE_NAME).withRecordEntity(new SharedAsset().withGroupId(1).withAssetId(4))); + assertThat(insertOutput.getRecords().get(0).getErrors()).isEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @Disabled("This needs fixed, but we're committing as-we are to move forwards") + void testUpdateAsset() throws QException + { + //////////////////////////////////////////////////////////////////////////////////////// + // make sure we can't update an Asset if we don't have a key that would let us see it // + //////////////////////////////////////////////////////////////////////////////////////// + { + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(Asset.TABLE_NAME).withRecord(new QRecord().withValue("id", 1).withValue("modifyDate", Instant.now()))); + assertThat(updateOutput.getRecords().get(0).getErrors()).isNotEmpty(); + } + + /////////////////////////////////////////////// + // and if we do have a key, we can update it // + /////////////////////////////////////////////// + { + QContext.getQSession().withSecurityKeyValues(new HashMap<>()); + QContext.getQSession().withSecurityKeyValue(USER_ID_KEY_TYPE, HOMER_ID); + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(Asset.TABLE_NAME).withRecord(new QRecord().withValue("id", 1).withValue("modifyDate", Instant.now()))); + assertThat(updateOutput.getRecords().get(0).getErrors()).isEmpty(); + } + } + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Asset.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Asset.java new file mode 100644 index 00000000..b911314d --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Asset.java @@ -0,0 +1,227 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.sharing.model; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; + + +/******************************************************************************* + ** QRecord Entity for Asset table + *******************************************************************************/ +public class Asset extends QRecordEntity +{ + public static final String TABLE_NAME = "Asset"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField() + private String name; + + @QField(possibleValueSourceName = User.TABLE_NAME) + private Integer userId; + + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public Asset() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public Asset(QRecord record) + { + populateFromQRecord(record); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public Asset withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public Asset withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public Asset withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public Asset withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public Integer getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(Integer userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public Asset withUserId(Integer userId) + { + this.userId = userId; + return (this); + } + + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Client.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Client.java new file mode 100644 index 00000000..01fd6827 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Client.java @@ -0,0 +1,192 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.sharing.model; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; + + +/******************************************************************************* + ** QRecord Entity for Client table + *******************************************************************************/ +public class Client extends QRecordEntity +{ + public static final String TABLE_NAME = "Client"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField() + private String name; + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public Client() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public Client(QRecord record) + { + populateFromQRecord(record); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public Client withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public Client withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public Client withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public Client withName(String name) + { + this.name = name; + return (this); + } + + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Group.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Group.java new file mode 100644 index 00000000..4c4ac65b --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/Group.java @@ -0,0 +1,226 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.sharing.model; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; + + +/******************************************************************************* + ** QRecord Entity for Group table + *******************************************************************************/ +public class Group extends QRecordEntity +{ + public static final String TABLE_NAME = "Group"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField() + private String name; + + @QField(possibleValueSourceName = Client.TABLE_NAME) + private Integer clientId; + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public Group() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public Group(QRecord record) + { + populateFromQRecord(record); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public Group withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public Group withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public Group withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public Group withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for clientId + *******************************************************************************/ + public Integer getClientId() + { + return (this.clientId); + } + + + + /******************************************************************************* + ** Setter for clientId + *******************************************************************************/ + public void setClientId(Integer clientId) + { + this.clientId = clientId; + } + + + + /******************************************************************************* + ** Fluent setter for clientId + *******************************************************************************/ + public Group withClientId(Integer clientId) + { + this.clientId = clientId; + return (this); + } + + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/SharedAsset.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/SharedAsset.java new file mode 100644 index 00000000..f0636525 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/SharedAsset.java @@ -0,0 +1,261 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.sharing.model; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; + + +/******************************************************************************* + ** QRecord Entity for SharedAsset table + *******************************************************************************/ +public class SharedAsset extends QRecordEntity +{ + public static final String TABLE_NAME = "SharedAsset"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = Asset.TABLE_NAME) + private Integer assetId; + + @QField(possibleValueSourceName = User.TABLE_NAME) + private Integer userId; + + @QField(possibleValueSourceName = Group.TABLE_NAME) + private Integer groupId; + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public SharedAsset() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public SharedAsset(QRecord record) + { + populateFromQRecord(record); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public SharedAsset withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public SharedAsset withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public SharedAsset withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for assetId + *******************************************************************************/ + public Integer getAssetId() + { + return (this.assetId); + } + + + + /******************************************************************************* + ** Setter for assetId + *******************************************************************************/ + public void setAssetId(Integer assetId) + { + this.assetId = assetId; + } + + + + /******************************************************************************* + ** Fluent setter for assetId + *******************************************************************************/ + public SharedAsset withAssetId(Integer assetId) + { + this.assetId = assetId; + return (this); + } + + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public Integer getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(Integer userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public SharedAsset withUserId(Integer userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for groupId + *******************************************************************************/ + public Integer getGroupId() + { + return (this.groupId); + } + + + + /******************************************************************************* + ** Setter for groupId + *******************************************************************************/ + public void setGroupId(Integer groupId) + { + this.groupId = groupId; + } + + + + /******************************************************************************* + ** Fluent setter for groupId + *******************************************************************************/ + public SharedAsset withGroupId(Integer groupId) + { + this.groupId = groupId; + return (this); + } + + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/User.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/User.java new file mode 100644 index 00000000..06de5745 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/sharing/model/User.java @@ -0,0 +1,193 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.sharing.model; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; + + +/******************************************************************************* + ** QRecord Entity for User table + *******************************************************************************/ +public class User extends QRecordEntity +{ + public static final String TABLE_NAME = "User"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField() + private String username; + + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public User() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public User(QRecord record) + { + populateFromQRecord(record); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public User withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public User withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public User withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for username + *******************************************************************************/ + public String getUsername() + { + return (this.username); + } + + + + /******************************************************************************* + ** Setter for username + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + *******************************************************************************/ + public User withUsername(String username) + { + this.username = username; + return (this); + } + + +} diff --git a/qqq-backend-module-rdbms/src/test/resources/prime-test-database-sharing-test.sql b/qqq-backend-module-rdbms/src/test/resources/prime-test-database-sharing-test.sql new file mode 100644 index 00000000..70a00914 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/resources/prime-test-database-sharing-test.sql @@ -0,0 +1,74 @@ +-- +-- QQQ - Low-code Application Framework for Engineers. +-- Copyright (C) 2021-2022. Kingsrook, LLC +-- 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States +-- contact@kingsrook.com +-- https://github.com/Kingsrook/ +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . +-- + +DROP TABLE IF EXISTS `user`; +CREATE TABLE `user` +( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + create_date TIMESTAMP DEFAULT NOW(), + modify_date TIMESTAMP DEFAULT NOW(), + username VARCHAR(100) +); + + +DROP TABLE IF EXISTS `group`; +CREATE TABLE `group` +( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + create_date TIMESTAMP DEFAULT NOW(), + modify_date TIMESTAMP DEFAULT NOW(), + name VARCHAR(100), + client_id INTEGER +); + + +DROP TABLE IF EXISTS `client`; +CREATE TABLE `client` +( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + create_date TIMESTAMP DEFAULT NOW(), + modify_date TIMESTAMP DEFAULT NOW(), + name VARCHAR(100) +); + + +DROP TABLE IF EXISTS asset; +CREATE TABLE asset +( + id INT AUTO_INCREMENT PRIMARY KEY, + create_date TIMESTAMP DEFAULT NOW(), + modify_date TIMESTAMP DEFAULT NOW(), + name VARCHAR(100), + user_id INTEGER +); + + +DROP TABLE IF EXISTS shared_asset; +CREATE TABLE shared_asset +( + id INT AUTO_INCREMENT PRIMARY KEY, + create_date TIMESTAMP DEFAULT NOW(), + modify_date TIMESTAMP DEFAULT NOW(), + asset_id INTEGER, + user_id INTEGER, + group_id INTEGER +); + diff --git a/qqq-bom/pom.xml b/qqq-bom/pom.xml new file mode 100644 index 00000000..7e3b3ac2 --- /dev/null +++ b/qqq-bom/pom.xml @@ -0,0 +1,98 @@ + + + + + 4.0.0 + + qqq-bom-pom + pom + + + com.kingsrook.qqq + qqq-parent-project + ${revision} + + + + + + com.kingsrook.qqq + qqq-backend-core + ${revision} + + + com.kingsrook.qqq + qqq-backend-module-rdbms + ${revision} + + + com.kingsrook.qqq + qqq-backend-module-mongodb + ${revision} + + + com.kingsrook.qqq + qqq-backend-module-api + ${revision} + + + com.kingsrook.qqq + qqq-backend-module-filesystem + ${revision} + + + com.kingsrook.qqq + qqq-middleware-javalin + ${revision} + + + com.kingsrook.qqq + qqq-middleware-slack + ${revision} + + + com.kingsrook.qqq + qqq-middleware-api + ${revision} + + + com.kingsrook.qqq + qqq-middleware-picocli + ${revision} + + + com.kingsrook.qqq + qqq-language-support-javascript + ${revision} + + + + + + + github-qqq-maven-registry + GitHub QQQ Maven Registry + https://maven.pkg.github.com/Kingsrook/qqq-maven-registry + + + + diff --git a/qqq-dev-tools/MODULE_LIST b/qqq-dev-tools/MODULE_LIST index cf9a6d91..156e3e08 100644 --- a/qqq-dev-tools/MODULE_LIST +++ b/qqq-dev-tools/MODULE_LIST @@ -9,3 +9,4 @@ qqq-middleware-picocli qqq-middleware-slack qqq-middleware-api qqq-frontend-material-dashboard +qqq-bom-pom diff --git a/qqq-dev-tools/bin/xbar-circleci-latest.sh b/qqq-dev-tools/bin/xbar-circleci-latest.sh index 0b6f92c7..4a39eb8d 100755 --- a/qqq-dev-tools/bin/xbar-circleci-latest.sh +++ b/qqq-dev-tools/bin/xbar-circleci-latest.sh @@ -14,12 +14,15 @@ . ~/.bashrc . $QQQ_DEV_TOOLS_DIR/.env -FILE=/tmp/cci.$$ +DIR=/tmp/xbar-circleci-latest +mkdir -p $DIR +FILE=$DIR/cci.$$ JQ=/opt/homebrew/bin/jq -curl -s -H "Circle-Token: ${CIRCLE_TOKEN}" "https://circleci.com/api/v1.1/recent-builds?limit=10&shallow=true" > $FILE +curl -s -H "Circle-Token: ${CIRCLE_TOKEN}" "https://circleci.com/api/v1.1/recent-builds?limit=50&shallow=true" > $FILE NOW=$(date +%s) needPipe=0 +displayedAny=0 checkBuild() { @@ -38,11 +41,11 @@ checkBuild() fi endDate=$($JQ ".[$i].stop_time" < $FILE | sed 's/"//g;s/null//;') - curl $avatarUrl > /tmp/avatar.jpg - sips -s dpiHeight 96 -s dpiWidth 96 /tmp/avatar.jpg -o /tmp/avatar-96dpi.jpg > /dev/null - sips -z 20 20 /tmp/avatar-96dpi.jpg -o /tmp/avatar-20.jpg > /dev/null - base64 -i /tmp/avatar-20.jpg > /tmp/avatar.b64 - avatarB64=$(cat /tmp/avatar.b64) + curl $avatarUrl > $DIR/avatar.jpg + sips -s dpiHeight 96 -s dpiWidth 96 $DIR/avatar.jpg -o $DIR/avatar-96dpi.jpg > /dev/null + sips -z 20 20 $DIR/avatar-96dpi.jpg -o $DIR/avatar-20.jpg > /dev/null + base64 -i $DIR/avatar-20.jpg > $DIR/avatar.b64 + avatarB64=$(cat $DIR/avatar.b64) shortRepo="$repo" case $repo in @@ -50,8 +53,10 @@ checkBuild() qqq-frontend-core) shortRepo="fc";; qqq-frontend-material-dashboard) shortRepo="qfmd";; ColdTrack-Live) shortRepo="ctl";; - ColdTrack-Live-Scripts) shortRepo="cls";; - Infoplus-Scripts) shortRepo="ips";; + ## ColdTrack-Live-Scripts) shortRepo="cls";; + ## Infoplus-Scripts) shortRepo="ips";; + ColdTrack-Live-Scripts) return;; + Infoplus-Scripts) return;; esac timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S%z" $(echo "$startDate" | sed 's/\....Z/+0000/') +%s) @@ -93,7 +98,9 @@ checkBuild() #################################################################################################### ## if this is the 1st build, or it was less than some-short-time ago, then put it in the menu bar ## #################################################################################################### - if [ $index -lt 1 -o $seconds -lt 600 ]; then + if [ $displayedAny -eq 0 -o $seconds -lt 600 -o "$buildStatus" == "running" ]; then + + displayedAny=1 ########################################################################################### ## put a pipe (unicode special pipe, to not break things) before all but the first build ## @@ -115,14 +122,18 @@ checkBuild() details="---" details="$details\n🔄 Refresh | refresh=true" -for i in $(seq 0 9); do +for i in $(seq 0 19); do checkBuild $i done +if [ $displayedAny -eq 0 ]; then + echo "🤷" +fi + ## echo "@$(date +%M:%S)" echo echo -e "$details" -cp $FILE /tmp/cci-latest.json +cp $FILE $DIR/cci-latest.json rm $FILE diff --git a/qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/TestUtils.java b/qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/TestUtils.java index a7eab8b5..a094c1c9 100644 --- a/qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/TestUtils.java +++ b/qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/TestUtils.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticat import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; /******************************************************************************* @@ -74,7 +75,7 @@ public class TestUtils { return (new QBackendMetaData() .withName(DEFAULT_BACKEND_NAME) - .withBackendType("memory")); + .withBackendType(MemoryBackendModule.class)); } diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java index 56444250..84137b7d 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java @@ -1000,6 +1000,7 @@ public class ApiImplementation ApiProcessInput apiProcessInput = apiProcessMetaData.getInput(); if(apiProcessInput != null) { + processProcessInputFields(paramMap, badRequestMessages, runProcessInput, apiProcessInput.getPathParams()); processProcessInputFields(paramMap, badRequestMessages, runProcessInput, apiProcessInput.getQueryStringParams()); processProcessInputFields(paramMap, badRequestMessages, runProcessInput, apiProcessInput.getFormParams()); processProcessInputFields(paramMap, badRequestMessages, runProcessInput, apiProcessInput.getObjectBodyParams()); @@ -1143,7 +1144,10 @@ public class ApiImplementation ApiProcessOutputInterface output = apiProcessMetaData.getOutput(); if(output != null) { - return (new HttpApiResponse(output.getSuccessStatusCode(runProcessInput, runProcessOutput), output.getOutputForProcess(runProcessInput, runProcessOutput))); + Serializable outputForProcess = output.getOutputForProcess(runProcessInput, runProcessOutput); + HttpApiResponse httpApiResponse = new HttpApiResponse(output.getSuccessStatusCode(runProcessInput, runProcessOutput), outputForProcess); + output.customizeHttpApiResponse(httpApiResponse, runProcessInput, runProcessOutput); + return httpApiResponse; } else { diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java index 303ffeff..7b42895b 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java @@ -194,6 +194,8 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction neededTableSchemas = new HashSet<>(); @@ -314,40 +316,40 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction new Path().withGet(methodForProcess); @@ -1046,16 +1066,28 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction parameters = new ArrayList<>(); + ApiProcessInput apiProcessInput = apiProcessMetaData.getInput(); + ApiProcessInputFieldsContainer pathParams = apiProcessInput.getPathParams(); + if(pathParams != null) + { + for(QFieldMetaData field : CollectionUtils.nonNullList(pathParams.getFields())) + { + parameters.add(processFieldToParameter(apiInstanceMetaData, field).withIn("path")); + } + } + + parameters.add(new Parameter() .withName("jobId") .withIn("path") .withRequired(true) .withDescription("Id of the job, as returned by the API call that started it.") - .withSchema(new Schema().withType("string").withFormat("uuid")) - )); + .withSchema(new Schema().withType("string").withFormat("uuid"))); + + //////////////////////////////////////////////////////// + // add the async input for optionally-async processes // + //////////////////////////////////////////////////////// + methodForProcess.setParameters(parameters); ////////////////////////////////// // build all possible responses // @@ -1126,7 +1158,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction ApiTableMetaDataContainer.of(associatedTable).getApiTableMetaData(apiName), new ApiTableMetaData()); + ApiTableMetaData associatedApiTableMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiTableMetaDataContainer.of(associatedTable).getApiTableMetaData(apiName), new ApiTableMetaData()); String associatedTableApiName = StringUtils.hasContent(associatedApiTableMetaData.getApiTableName()) ? associatedApiTableMetaData.getApiTableName() : associatedTableName; ApiAssociationMetaData apiAssociationMetaData = thisApiTableMetaData.getApiAssociationMetaData().get(association.getName()); @@ -1411,14 +1443,14 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction example = switch(method.toLowerCase()) @@ -1680,13 +1711,12 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction "string"; - case INTEGER -> "integer"; + case INTEGER, LONG -> "integer"; // todo - we could give 'format' w/ int32 & int64 to further specify case DECIMAL -> "number"; case BOOLEAN -> "boolean"; }; @@ -1707,7 +1737,6 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction. + */ + +package com.kingsrook.qqq.api.implementations.savedreports; + + +import com.kingsrook.qqq.api.model.metadata.processes.PreRunApiProcessCustomizer; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; + + +/******************************************************************************* + ** API-Customizer for the RenderSavedReport process + *******************************************************************************/ +public class RenderSavedReportProcessApiCustomizer implements PreRunApiProcessCustomizer +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preApiRun(RunProcessInput runProcessInput) throws QException + { + Integer reportId = runProcessInput.getValueInteger("reportId"); + if(reportId != null) + { + QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(reportId)); + if(record == null) + { + throw (new QNotFoundException("Report Id " + reportId + " was not found.")); + } + + runProcessInput.setCallback(QProcessCallbackFactory.forPrimaryKey("id", reportId)); + } + } + +} diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiMetaDataEnricher.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiMetaDataEnricher.java new file mode 100644 index 00000000..528adfe8 --- /dev/null +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiMetaDataEnricher.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.api.implementations.savedreports; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData; +import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessCustomizers; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInput; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInputFieldsContainer; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessMetaData; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessMetaDataContainer; +import com.kingsrook.qqq.api.model.openapi.ExampleWithListValue; +import com.kingsrook.qqq.api.model.openapi.ExampleWithSingleValue; +import com.kingsrook.qqq.api.model.openapi.HttpMethod; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; + + +/******************************************************************************* + ** Class that helps prepare the RenderSavedReport process for use in an API + *******************************************************************************/ +public class RenderSavedReportProcessApiMetaDataEnricher +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static ApiProcessMetaData setupProcessForApi(QProcessMetaData process, String apiName, String initialApiVersion) + { + ApiProcessMetaDataContainer apiProcessMetaDataContainer = ApiProcessMetaDataContainer.ofOrWithNew(process); + + ApiProcessInput input = new ApiProcessInput() + .withPathParams(new ApiProcessInputFieldsContainer() + .withField(new QFieldMetaData("reportId", QFieldType.INTEGER) + .withIsRequired(true) + .withSupplementalMetaData(newDefaultApiFieldMetaData("Saved Report Id", 1701)))) + .withQueryStringParams(new ApiProcessInputFieldsContainer() + .withField(new QFieldMetaData("reportFormat", QFieldType.STRING) + .withIsRequired(true) + .withPossibleValueSourceName(ReportFormatPossibleValueEnum.NAME) + .withSupplementalMetaData(newDefaultApiFieldMetaData("Requested file format", "XLSX")))); + // todo (when implemented) - probably a JSON doc w/ input values. + + RenderSavedReportProcessApiProcessOutput output = new RenderSavedReportProcessApiProcessOutput(); + + ApiProcessMetaData apiProcessMetaData = new ApiProcessMetaData() + .withInitialVersion(initialApiVersion) + .withCustomizer(ApiProcessCustomizers.PRE_RUN.getRole(), new QCodeReference(RenderSavedReportProcessApiCustomizer.class)) + .withAsyncMode(ApiProcessMetaData.AsyncMode.OPTIONAL) + .withMethod(HttpMethod.GET) + .withInput(input) + .withOutput(output); + + apiProcessMetaDataContainer.withApiProcessMetaData(apiName, apiProcessMetaData); + + return (apiProcessMetaData); + } + + + + /******************************************************************************* + ** todo - move to higher-level utility + *******************************************************************************/ + public static ApiFieldMetaDataContainer newDefaultApiFieldMetaData(String description, Serializable example) + { + ApiFieldMetaData defaultApiFieldMetaData = new ApiFieldMetaData().withDescription(description); + ApiFieldMetaDataContainer apiFieldMetaDataContainer = new ApiFieldMetaDataContainer().withDefaultApiFieldMetaData(defaultApiFieldMetaData); + if(example instanceof List) + { + defaultApiFieldMetaData.withExample(new ExampleWithListValue().withValue((List) example)); + } + else + { + defaultApiFieldMetaData.withExample(new ExampleWithSingleValue().withValue(example)); + } + + return (apiFieldMetaDataContainer); + } + +} diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiProcessOutput.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiProcessOutput.java new file mode 100644 index 00000000..f2429917 --- /dev/null +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiProcessOutput.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.api.implementations.savedreports; + + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; +import com.kingsrook.qqq.api.model.actions.HttpApiResponse; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessOutputInterface; +import com.kingsrook.qqq.api.model.openapi.Content; +import com.kingsrook.qqq.api.model.openapi.Response; +import com.kingsrook.qqq.api.model.openapi.Schema; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import org.eclipse.jetty.http.HttpStatus; + + +/******************************************************************************* + ** api process output specifier for the RenderSavedReport process + *******************************************************************************/ +public class RenderSavedReportProcessApiProcessOutput implements ApiProcessOutputInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Serializable getOutputForProcess(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws QException + { + ////////////////////////////////////////////////////////////////// + // we don't use output like this - see customizeHttpApiResponse // + ////////////////////////////////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void customizeHttpApiResponse(HttpApiResponse httpApiResponse, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // we don't need anyone else to format our response - assume that we've done so ourselves. // + ///////////////////////////////////////////////////////////////////////////////////////////// + httpApiResponse.setNeedsFormattedAsJson(false); + + ///////////////////////////////////////////// + // set content type based on report format // + ///////////////////////////////////////////// + ReportFormat reportFormat = ReportFormat.fromString(runProcessOutput.getValueString("reportFormat")); + httpApiResponse.setContentType(reportFormat.getMimeType()); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // get an input stream from the backend where the report content is stored - send that down to the caller // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + httpApiResponse.setInputStream(new StorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public HttpStatus.Code getSuccessStatusCode(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) + { + return (HttpStatus.Code.OK); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Map getSpecResponses(String apiName) + { + Map contentMap = new LinkedHashMap<>(); + contentMap.put(ReportFormat.JSON.getMimeType(), new Content() + .withSchema(new Schema() + .withDescription("JSON Report contents") + .withExample(""" + [ + {"id": 1, "name": "James"}, + {"id": 2, "name": "Jean-Luc"} + ] + """) + .withType("string") + .withFormat("text"))); + + contentMap.put(ReportFormat.CSV.getMimeType(), new Content() + .withSchema(new Schema() + .withDescription("CSV Report contents") + .withExample(""" + "id","name" + 1,"James" + 2,"Jean-Luc" + """) + .withType("string") + .withFormat("text"))); + + contentMap.put(ReportFormat.XLSX.getMimeType(), new Content() + .withSchema(new Schema() + .withDescription("Excel Report contents") + .withType("string") + .withFormat("binary"))); + + return Map.of(HttpStatus.Code.OK.getCode(), new Response() + .withDescription("Report contents in the requested format.") + .withContent(contentMap)); + } + +} diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java index cd99af77..ed767327 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java @@ -298,7 +298,6 @@ public class QJavalinApiHandler /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") private void runProcess(Context context, QProcessMetaData processMetaData, ApiProcessMetaData apiProcessMetaData, ApiInstanceMetaData apiInstanceMetaData) { String version = context.pathParam("version"); @@ -316,6 +315,7 @@ public class QJavalinApiHandler ApiProcessInput input = apiProcessMetaData.getInput(); if(input != null) { + processProcessInputFieldsContainer(context, parameters, input.getPathParams(), Context::pathParam); processProcessInputFieldsContainer(context, parameters, input.getQueryStringParams(), Context::queryParam); processProcessInputFieldsContainer(context, parameters, input.getFormParams(), Context::formParam); @@ -347,9 +347,7 @@ public class QJavalinApiHandler ////////////////// QJavalinAccessLogger.logEndSuccess(); context.status(response.getStatusCode().getCode()); - String resultString = toJson(Objects.requireNonNullElse(response.getResponseBodyObject(), "")); - context.result(resultString); - storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); + handleProcessResponse(context, response, apiLog); } catch(Exception e) { @@ -360,6 +358,63 @@ public class QJavalinApiHandler + /******************************************************************************* + ** + *******************************************************************************/ + private static void handleProcessResponse(Context context, HttpApiResponse response, APILog apiLog) + { + if(response.getNeedsFormattedAsJson()) + { + ///////////////////////////////////////////////////////////////////////////////////////// + // if the response object says that we should format the response as json, then do so. // + ///////////////////////////////////////////////////////////////////////////////////////// + String resultString = toJson(Objects.requireNonNullElse(response.getResponseBodyObject(), "")); + context.result(resultString); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); + } + else + { + if(StringUtils.hasContent(response.getContentType())) + { + context.contentType(response.getContentType()); + } + + /////////////////////////////////////////////////////////////////////////////////// + // if there's an input stream in the response, just send that down to the client // + /////////////////////////////////////////////////////////////////////////////////// + if(response.getInputStream() != null) + { + context.result(response.getInputStream()); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody("Streamed result")); + } + else + { + //////////////////////////////////////////////////////////////////////////////////// + // else, try to return it raw - as byte[], or String, or as a converted-to-String // + //////////////////////////////////////////////////////////////////////////////////// + Serializable result = Objects.requireNonNullElse(response.getResponseBodyObject(), ""); + if(result instanceof byte[] ba) + { + context.result(ba); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody("Byte array of length: " + ba.length)); + } + else if(result instanceof String s) + { + context.result(s); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(s)); + } + else + { + String resultString = String.valueOf(result); + context.result(resultString); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -381,9 +436,7 @@ public class QJavalinApiHandler ////////////////// QJavalinAccessLogger.logEndSuccess(); context.status(response.getStatusCode().getCode()); - String resultString = toJson(Objects.requireNonNullElse(response.getResponseBodyObject(), "")); - context.result(resultString); - storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); + handleProcessResponse(context, response, apiLog); } catch(Exception e) { diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/APILog.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/APILog.java index 8f6cbcc3..3e8352eb 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/APILog.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/APILog.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QIgnore; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -425,6 +426,7 @@ public class APILog extends QRecordEntity /******************************************************************************* ** Getter for securityKeyValues *******************************************************************************/ + @QIgnore public Map getSecurityKeyValues() { return (this.securityKeyValues); diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/HttpApiResponse.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/HttpApiResponse.java index 5d3b1c66..8bedcda0 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/HttpApiResponse.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/HttpApiResponse.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.api.model.actions; +import java.io.InputStream; import java.io.Serializable; import org.eclipse.jetty.http.HttpStatus; @@ -35,6 +36,16 @@ public class HttpApiResponse private HttpStatus.Code statusCode; private Serializable responseBodyObject; + private String contentType; + + private InputStream inputStream; + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // by default - QJavalinApiHandler will format the responseBodyObject as JSON. // + // set this field to false if you don't want it to do that (e.g., if your response is, say, a byte[]) // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + private boolean needsFormattedAsJson = true; + /******************************************************************************* @@ -119,4 +130,97 @@ public class HttpApiResponse return (this); } + + + /******************************************************************************* + ** Getter for needsFormattedAsJson + *******************************************************************************/ + public boolean getNeedsFormattedAsJson() + { + return (this.needsFormattedAsJson); + } + + + + /******************************************************************************* + ** Setter for needsFormattedAsJson + *******************************************************************************/ + public void setNeedsFormattedAsJson(boolean needsFormattedAsJson) + { + this.needsFormattedAsJson = needsFormattedAsJson; + } + + + + /******************************************************************************* + ** Fluent setter for needsFormattedAsJson + *******************************************************************************/ + public HttpApiResponse withNeedsFormattedAsJson(boolean needsFormattedAsJson) + { + this.needsFormattedAsJson = needsFormattedAsJson; + return (this); + } + + + + /******************************************************************************* + ** Getter for contentType + *******************************************************************************/ + public String getContentType() + { + return (this.contentType); + } + + + + /******************************************************************************* + ** Setter for contentType + *******************************************************************************/ + public void setContentType(String contentType) + { + this.contentType = contentType; + } + + + + /******************************************************************************* + ** Fluent setter for contentType + *******************************************************************************/ + public HttpApiResponse withContentType(String contentType) + { + this.contentType = contentType; + return (this); + } + + + /******************************************************************************* + ** Getter for inputStream + *******************************************************************************/ + public InputStream getInputStream() + { + return (this.inputStream); + } + + + + /******************************************************************************* + ** Setter for inputStream + *******************************************************************************/ + public void setInputStream(InputStream inputStream) + { + this.inputStream = inputStream; + } + + + + /******************************************************************************* + ** Fluent setter for inputStream + *******************************************************************************/ + public HttpApiResponse withInputStream(InputStream inputStream) + { + this.inputStream = inputStream; + return (this); + } + + } \ No newline at end of file diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java index 71fcac0a..2ea76133 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java @@ -40,6 +40,17 @@ public class ApiInstanceMetaDataContainer extends QSupplementalInstanceMetaData + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getName() + { + return ApiSupplementType.NAME; + } + + + /******************************************************************************* ** Constructor ** diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInput.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInput.java index 83b05691..7109abbb 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInput.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInput.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; *******************************************************************************/ public class ApiProcessInput { + private ApiProcessInputFieldsContainer pathParams; private ApiProcessInputFieldsContainer queryStringParams; private ApiProcessInputFieldsContainer formParams; private ApiProcessInputFieldsContainer recordBodyParams; @@ -44,6 +45,11 @@ public class ApiProcessInput *******************************************************************************/ public String getRecordIdsParamName() { + if(pathParams != null && pathParams.getRecordIdsField() != null) + { + return (pathParams.getRecordIdsField().getName()); + } + if(queryStringParams != null && queryStringParams.getRecordIdsField() != null) { return (queryStringParams.getRecordIdsField().getName()); @@ -217,4 +223,35 @@ public class ApiProcessInput return (this); } + + /******************************************************************************* + ** Getter for pathParams + *******************************************************************************/ + public ApiProcessInputFieldsContainer getPathParams() + { + return (this.pathParams); + } + + + + /******************************************************************************* + ** Setter for pathParams + *******************************************************************************/ + public void setPathParams(ApiProcessInputFieldsContainer pathParams) + { + this.pathParams = pathParams; + } + + + + /******************************************************************************* + ** Fluent setter for pathParams + *******************************************************************************/ + public ApiProcessInput withPathParams(ApiProcessInputFieldsContainer pathParams) + { + this.pathParams = pathParams; + return (this); + } + + } diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessOutputInterface.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessOutputInterface.java index 6e430aa4..1ee3e70d 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessOutputInterface.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessOutputInterface.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.api.model.metadata.processes; import java.io.Serializable; import java.util.Map; +import com.kingsrook.qqq.api.model.actions.HttpApiResponse; import com.kingsrook.qqq.api.model.openapi.Response; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; @@ -61,4 +62,13 @@ public interface ApiProcessOutputInterface .withDescription("Process has been successfully executed.") )); } + + /******************************************************************************* + ** + *******************************************************************************/ + default void customizeHttpApiResponse(HttpApiResponse httpApiResponse, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws QException + { + + } + } diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessSummaryListOutput.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessSummaryListOutput.java index 847e4b8c..9abf7c0f 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessSummaryListOutput.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessSummaryListOutput.java @@ -233,7 +233,6 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface *******************************************************************************/ private static String getResultMapMessagePrefix(ProcessSummaryLineInterface processSummaryLine) { - @SuppressWarnings("checkstyle:indentation") String messagePrefix = switch(processSummaryLine.getStatus()) { case OK, INFO, ERROR -> ""; @@ -251,7 +250,6 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface { HashMap map = new HashMap<>(); - @SuppressWarnings("checkstyle:indentation") HttpStatus.Code code = switch(processSummaryLine.getStatus()) { case OK, WARNING, INFO -> HttpStatus.Code.OK; diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessUtils.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessUtils.java index dc75f7ad..b5cba76d 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessUtils.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessUtils.java @@ -34,9 +34,11 @@ import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.commons.lang.BooleanUtils; @@ -162,9 +164,18 @@ public class ApiProcessUtils *******************************************************************************/ public static String getProcessApiPath(QInstance qInstance, QProcessMetaData process, ApiProcessMetaData apiProcessMetaData, ApiInstanceMetaData apiInstanceMetaData) { + StringBuilder pathParams = new StringBuilder(); + if(ObjectUtils.ifCan(() -> CollectionUtils.nullSafeHasContents(apiProcessMetaData.getInput().getPathParams().getFields()))) + { + for(QFieldMetaData field : apiProcessMetaData.getInput().getPathParams().getFields()) + { + pathParams.append("/{").append(field.getName()).append("}"); + } + } + if(StringUtils.hasContent(apiProcessMetaData.getPath())) { - return apiProcessMetaData.getPath() + "/" + apiProcessMetaData.getApiProcessName(); + return apiProcessMetaData.getPath() + "/" + apiProcessMetaData.getApiProcessName() + pathParams; } else if(StringUtils.hasContent(process.getTableName())) { @@ -182,11 +193,11 @@ public class ApiProcessUtils } } } - return tablePathPart + "/" + apiProcessMetaData.getApiProcessName(); + return tablePathPart + "/" + apiProcessMetaData.getApiProcessName() + pathParams; } else { - return apiProcessMetaData.getApiProcessName(); + return apiProcessMetaData.getApiProcessName() + pathParams; } } diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/BaseTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/BaseTest.java index 2d7fd5b3..e77c0119 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/BaseTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/BaseTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.api; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -56,7 +57,7 @@ public class BaseTest ** *******************************************************************************/ @BeforeEach - void baseBeforeEach() + void baseBeforeEach() throws QException { QContext.init(TestUtils.defineInstance(), new QSession()); } diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java index 1cc32785..66cb753c 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.api; import java.util.List; import java.util.function.Consumer; +import com.kingsrook.qqq.api.implementations.savedreports.RenderSavedReportProcessApiMetaDataEnricher; import com.kingsrook.qqq.api.model.APIVersion; import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData; import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer; @@ -45,6 +46,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; @@ -71,7 +73,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMeta import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; @@ -79,6 +84,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.Mem import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -112,7 +118,7 @@ public class TestUtils /******************************************************************************* ** *******************************************************************************/ - public static QInstance defineInstance() + public static QInstance defineInstance() throws QException { QInstance qInstance = new QInstance(); @@ -133,6 +139,8 @@ public class TestUtils qInstance.setAuthentication(new Auth0AuthenticationMetaData().withType(QAuthenticationType.FULLY_ANONYMOUS).withName("anonymous")); + addSavedReports(qInstance); + qInstance.withSupplementalMetaData(new ApiInstanceMetaDataContainer() .withApiInstanceMetaData(new ApiInstanceMetaData() .withName(API_NAME) @@ -161,6 +169,18 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + private static void addSavedReports(QInstance qInstance) throws QException + { + qInstance.add(TablesPossibleValueSourceMetaDataProvider.defineTablesPossibleValueSource(qInstance)); + new SavedReportsMetaDataProvider().defineAll(qInstance, MEMORY_BACKEND_NAME, MEMORY_BACKEND_NAME, null); + RenderSavedReportProcessApiMetaDataEnricher.setupProcessForApi(qInstance.getProcess(RenderSavedReportMetaDataProducer.NAME), API_NAME, V2022_Q4); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -531,6 +551,19 @@ public class TestUtils } + /******************************************************************************* + ** + *******************************************************************************/ + public static Integer insertSavedReport(SavedReport savedReport) throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(SavedReport.TABLE_NAME); + insertInput.setRecords(List.of(savedReport.toQRecord())); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + return insertOutput.getRecords().get(0).getValueInteger("id"); + } + + /******************************************************************************* ** diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TransformPersonStep.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TransformPersonStep.java index d3dfdd0a..a287370d 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TransformPersonStep.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TransformPersonStep.java @@ -61,7 +61,7 @@ public class TransformPersonStep extends AbstractTransformStep ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { for(QRecord record : runBackendStepInput.getRecords()) { diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java index 680c850d..c884d47c 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java @@ -25,7 +25,6 @@ package com.kingsrook.qqq.api.javalin; import com.kingsrook.qqq.api.BaseTest; import com.kingsrook.qqq.api.TestUtils; import com.kingsrook.qqq.api.actions.ApiImplementation; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; @@ -61,7 +60,7 @@ class QJavalinApiHandlerPermissionsTest extends BaseTest ** *******************************************************************************/ @BeforeAll - static void beforeAll() throws QInstanceValidationException + static void beforeAll() throws Exception { QInstance qInstance = TestUtils.defineInstance(); diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java index f495f51c..20fc10b1 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java @@ -36,7 +36,6 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; @@ -51,6 +50,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; @@ -66,6 +68,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.api.TestUtils.insertPersonRecord; +import static com.kingsrook.qqq.api.TestUtils.insertSavedReport; import static com.kingsrook.qqq.api.TestUtils.insertSimpsons; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -95,7 +98,7 @@ class QJavalinApiHandlerTest extends BaseTest ** *******************************************************************************/ @BeforeAll - static void beforeAll() throws QInstanceValidationException + static void beforeAll() throws Exception { QInstance qInstance = TestUtils.defineInstance(); @@ -1404,6 +1407,51 @@ class QJavalinApiHandlerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetProcessRenderSavedReport() throws QException + { + insertSimpsons(); + Integer reportId = insertSavedReport(new SavedReport() + .withLabel("Person Report") + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("lastName")))); + + HttpResponse response = Unirest.get(BASE_URL + "/api/" + VERSION + "/savedReport/renderSavedReport/" + reportId + "?reportFormat=CSV").asString(); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getHeaders().getFirst("content-type")).contains("csv"); + assertEquals(""" + "Id","First Name","Last Name" + "1","Homer","Simpson" + "2","Marge","Simpson" + "3","Bart","Simpson" + "4","Lisa","Simpson" + "5","Maggie","Simpson" + """, response.getBody()); + + response = Unirest.get(BASE_URL + "/api/" + VERSION + "/savedReport/renderSavedReport/" + reportId + "?reportFormat=JSON").asString(); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getHeaders().getFirst("content-type")).contains("json"); + JSONArray jsonArray = new JSONArray(response.getBody()); + assertEquals(5, jsonArray.length()); + assertThat(jsonArray.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Homer") + .hasFieldOrPropertyWithValue("lastName", "Simpson"); + + response = Unirest.get(BASE_URL + "/api/" + VERSION + "/savedReport/renderSavedReport/" + reportId + "?reportFormat=XLSX").asString(); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getHeaders().getFirst("content-type")).contains("openxmlformats-officedocument.spreadsheetml"); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 94ae0e71..ba87ccf1 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -32,7 +32,9 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -77,6 +79,7 @@ import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; @@ -100,6 +103,7 @@ import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSo import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; @@ -107,6 +111,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendVariant; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -279,6 +284,14 @@ public class QJavalinImplementation try { + //////////////////////////////////////////////////////////////////////////////// + // clear the cache of classes in this class, so that new classes can be found // + //////////////////////////////////////////////////////////////////////////////// + MetaDataProducerHelper.clearTopLevelClassCache(); + + ///////////////////////////////////////////////// + // try to get a new instance from the supplier // + ///////////////////////////////////////////////// QInstance newQInstance = qInstanceHotSwapSupplier.get(); if(newQInstance == null) { @@ -286,6 +299,9 @@ public class QJavalinImplementation return; } + /////////////////////////////////////////////////////////////////////////////////// + // validate the instance, and only if it passes, then set it in our static field // + /////////////////////////////////////////////////////////////////////////////////// new QInstanceValidator().validate(newQInstance); QJavalinImplementation.qInstance = newQInstance; LOG.info("Swapped qInstance"); @@ -364,8 +380,8 @@ public class QJavalinImplementation post("/export", QJavalinImplementation::dataExportWithoutFilename); get("/export/{filename}", QJavalinImplementation::dataExportWithFilename); post("/export/{filename}", QJavalinImplementation::dataExportWithFilename); - get("/possibleValues/{fieldName}", QJavalinImplementation::possibleValues); - post("/possibleValues/{fieldName}", QJavalinImplementation::possibleValues); + get("/possibleValues/{fieldName}", QJavalinImplementation::possibleValuesForTableField); + post("/possibleValues/{fieldName}", QJavalinImplementation::possibleValuesForTableField); // todo - add put and/or patch at this level (without a primaryKey) to do a bulk update based on primaryKeys in the records. path("/{primaryKey}", () -> @@ -382,6 +398,9 @@ public class QJavalinImplementation }); }); + get("/possibleValues/{possibleValueSourceName}", QJavalinImplementation::possibleValuesStandalone); + post("/possibleValues/{possibleValueSourceName}", QJavalinImplementation::possibleValuesStandalone); + get("/widget/{name}", QJavalinImplementation::widget); // todo - can we just do a slow log here? get("/serverInfo", QJavalinImplementation::serverInfo); @@ -427,7 +446,17 @@ public class QJavalinImplementation QSession session = authenticationModule.createSession(qInstance, authContext); context.cookie(SESSION_UUID_COOKIE_NAME, session.getUuid(), SESSION_COOKIE_AGE); - context.result(JsonUtils.toJson(MapBuilder.of("uuid", session.getUuid()))); + + Map resultMap = new HashMap<>(); + resultMap.put("uuid", session.getUuid()); + + if(session.getValuesForFrontend() != null) + { + LinkedHashMap valuesForFrontend = new LinkedHashMap<>(session.getValuesForFrontend()); + resultMap.put("values", valuesForFrontend); + } + + context.result(JsonUtils.toJson(resultMap)); } catch(Exception e) { @@ -710,10 +739,15 @@ public class QJavalinImplementation { throw (new QUserFacingException("Error updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getErrors()))); } - if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) - { - throw (new QUserFacingException("Warning updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); - } + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // at one time, we threw upon warning - but // + // on insert we need to return the record (e.g., to get a generated id), so, make update do the same. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) + // { + // throw (new QUserFacingException("Warning updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); + // } QJavalinAccessLogger.logEndSuccess(); context.result(JsonUtils.toJson(updateOutput)); @@ -779,9 +813,34 @@ public class QJavalinImplementation { String fieldName = formParam.getKey(); List values = formParam.getValue(); + if(CollectionUtils.nullSafeHasContents(values)) { String value = values.get(0); + + if("associations".equals(fieldName) && StringUtils.hasContent(value)) + { + JSONObject associationsJSON = new JSONObject(value); + for(String key : associationsJSON.keySet()) + { + JSONArray associatedRecordsJSON = associationsJSON.getJSONArray(key); + List associatedRecords = new ArrayList<>(); + record.withAssociatedRecords(key, associatedRecords); + + for(int i = 0; i < associatedRecordsJSON.length(); i++) + { + QRecord associatedRecord = new QRecord(); + JSONObject recordJSON = associatedRecordsJSON.getJSONObject(i); + for(String k : recordJSON.keySet()) + { + associatedRecord.withValue(k, ValueUtils.getValueAsString(recordJSON.get(k))); + } + associatedRecords.add(associatedRecord); + } + } + continue; + } + if(StringUtils.hasContent(value)) { record.setValue(fieldName, value); @@ -793,7 +852,6 @@ public class QJavalinImplementation } else { - // is this ever hit? record.setValue(fieldName, null); } } @@ -881,10 +939,16 @@ public class QJavalinImplementation { throw (new QUserFacingException("Error inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getErrors()))); } - if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) - { - throw (new QUserFacingException("Warning inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); - } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // at one time, we threw upon warning - but // + // our use-case is, the frontend, it wants to get the record, and show a success (with the generated id) // + // and then to also show a warning message - so - let it all be returned and handled on the frontend. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) + // { + // throw (new QUserFacingException("Warning inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); + // } QJavalinAccessLogger.logEndSuccess(logPair("primaryKey", () -> (outputRecord.getValue(table.getPrimaryKeyField())))); context.result(JsonUtils.toJson(insertOutput)); @@ -1196,7 +1260,9 @@ public class QJavalinImplementation } if(filter != null) { - queryInput.setFilter(JsonUtils.toObject(filter, QQueryFilter.class)); + QQueryFilter qQueryFilter = JsonUtils.toObject(filter, QQueryFilter.class); + queryInput.setFilter(qQueryFilter); + qQueryFilter.interpretValues(Collections.emptyMap()); } Integer skip = QJavalinUtils.integerQueryParam(context, "skip"); @@ -1460,10 +1526,11 @@ public class QJavalinImplementation setupSession(context, exportInput); exportInput.setTableName(tableName); - exportInput.setReportFormat(reportFormat); String filename = optionalFilename.orElse(tableName + "." + reportFormat.toString().toLowerCase(Locale.ROOT)); - exportInput.setFilename(filename); + exportInput.withReportDestination(new ReportDestination() + .withReportFormat(reportFormat) + .withFilename(filename)); Integer limit = QJavalinUtils.integerQueryParam(context, "limit"); exportInput.setLimit(limit); @@ -1494,7 +1561,7 @@ public class QJavalinImplementation UnsafeFunction preAction = (PipedOutputStream pos) -> { - exportInput.setReportOutputStream(pos); + exportInput.getReportDestination().setReportOutputStream(pos); ExportAction exportAction = new ExportAction(); exportAction.preExecute(exportInput); @@ -1650,9 +1717,9 @@ public class QJavalinImplementation /******************************************************************************* - ** + ** handler for a PVS that's associated with a field on a table. *******************************************************************************/ - private static void possibleValues(Context context) + private static void possibleValuesForTableField(Context context) { try { @@ -1691,36 +1758,71 @@ public class QJavalinImplementation /******************************************************************************* - ** + ** handler for a standalone (e.g., outside of a table or process) PVS. + *******************************************************************************/ + private static void possibleValuesStandalone(Context context) + { + try + { + String possibleValueSourceName = context.pathParam("possibleValueSourceName"); + + QPossibleValueSource pvs = qInstance.getPossibleValueSource(possibleValueSourceName); + if(pvs == null) + { + throw (new QNotFoundException("Could not find possible value source " + possibleValueSourceName + " in this instance.")); + } + + finishPossibleValuesRequest(context, possibleValueSourceName, null); + } + catch(Exception e) + { + handleException(context, e); + } + } + + + + /******************************************************************************* + ** continuation for table or process PVS's, *******************************************************************************/ static void finishPossibleValuesRequest(Context context, QFieldMetaData field) throws IOException, QException + { + QQueryFilter defaultQueryFilter = null; + if(field.getPossibleValueSourceFilter() != null) + { + Map values = new HashMap<>(); + if(context.formParamMap().containsKey("values")) + { + List valuesParamList = context.formParamMap().get("values"); + if(CollectionUtils.nullSafeHasContents(valuesParamList)) + { + String valuesParam = valuesParamList.get(0); + values = JsonUtils.toObject(valuesParam, Map.class); + } + } + + defaultQueryFilter = field.getPossibleValueSourceFilter().clone(); + defaultQueryFilter.interpretValues(values); + } + + finishPossibleValuesRequest(context, field.getPossibleValueSourceName(), defaultQueryFilter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static void finishPossibleValuesRequest(Context context, String possibleValueSourceName, QQueryFilter defaultFilter) throws IOException, QException { String searchTerm = context.queryParam("searchTerm"); String ids = context.queryParam("ids"); - Map values = new HashMap<>(); - if(context.formParamMap().containsKey("values")) - { - List valuesParamList = context.formParamMap().get("values"); - if(CollectionUtils.nullSafeHasContents(valuesParamList)) - { - String valuesParam = valuesParamList.get(0); - values = JsonUtils.toObject(valuesParam, Map.class); - } - } - SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); setupSession(context, input); - input.setPossibleValueSourceName(field.getPossibleValueSourceName()); + input.setPossibleValueSourceName(possibleValueSourceName); input.setSearchTerm(searchTerm); - if(field.getPossibleValueSourceFilter() != null) - { - QQueryFilter filter = field.getPossibleValueSourceFilter().clone(); - filter.interpretValues(values); - input.setDefaultQueryFilter(filter); - } - if(StringUtils.hasContent(ids)) { List idList = new ArrayList<>(Arrays.asList(ids.split(","))); @@ -1732,6 +1834,7 @@ public class QJavalinImplementation Map result = new HashMap<>(); result.put("options", output.getResults()); context.result(JsonUtils.toJson(result)); + } @@ -1876,4 +1979,13 @@ public class QJavalinImplementation MILLIS_BETWEEN_HOT_SWAPS = millisBetweenHotSwaps; } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static long getStartTimeMillis() + { + return (startTime); + } } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 941c9f24..5faa8c08 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -45,10 +45,12 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.async.JobGoingAsyncException; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessAction; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; @@ -60,15 +62,18 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -126,6 +131,7 @@ public class QJavalinProcessHandler post("/step/{step}", QJavalinProcessHandler::processStep); get("/status/{jobUUID}", QJavalinProcessHandler::processStatus); get("/records", QJavalinProcessHandler::processRecords); + get("/cancel", QJavalinProcessHandler::processCancel); }); get("/possibleValues/{fieldName}", QJavalinProcessHandler::possibleValues); @@ -203,10 +209,12 @@ public class QJavalinProcessHandler QJavalinImplementation.setupSession(context, reportInput); PermissionsHelper.checkReportPermissionThrowing(reportInput, reportName); - reportInput.setReportFormat(reportFormat); reportInput.setReportName(reportName); reportInput.setInputValues(null); // todo! - reportInput.setFilename(filename); + + reportInput.setReportDestination(new ReportDestination() + .withReportFormat(reportFormat) + .withFilename(filename)); ////////////////////////////////////////////////////////////// // process the report's input fields, from the query string // @@ -239,7 +247,7 @@ public class QJavalinProcessHandler UnsafeFunction preAction = (PipedOutputStream pos) -> { - reportInput.setReportOutputStream(pos); + reportInput.getReportDestination().setReportOutputStream(pos); GenerateReportAction reportAction = new GenerateReportAction(); // any pre-action?? export uses this for "too many rows" checks... @@ -282,12 +290,24 @@ public class QJavalinProcessHandler // todo context.contentType(reportFormat.getMimeType()); context.header("Content-Disposition", "filename=" + context.pathParam("file")); - String filePath = context.queryParam("filePath"); - if(filePath == null) + String filePath = context.queryParam("filePath"); + String storageTableName = context.queryParam("storageTableName"); + String reference = context.queryParam("storageReference"); + + if(filePath != null) { - throw (new QBadRequestException("A filePath was not provided.")); + context.result(new FileInputStream(filePath)); } - context.result(new FileInputStream(filePath)); + else if(storageTableName != null && reference != null) + { + InputStream inputStream = new StorageAction().getInputStream(new StorageInput(storageTableName).withReference(reference)); + context.result(inputStream); + } + else + { + throw (new QBadRequestException("Missing query parameters to identify file to download")); + } + } catch(Exception e) { @@ -431,6 +451,12 @@ public class QJavalinProcessHandler } resultForCaller.put("values", runProcessOutput.getValues()); runProcessOutput.getProcessState().getNextStepName().ifPresent(nextStep -> resultForCaller.put("nextStep", nextStep)); + + List updatedFrontendStepList = runProcessOutput.getUpdatedFrontendStepList(); + if(updatedFrontendStepList != null) + { + resultForCaller.put("updatedFrontendStepList", updatedFrontendStepList); + } } @@ -744,6 +770,32 @@ public class QJavalinProcessHandler + /******************************************************************************* + ** + *******************************************************************************/ + private static void processCancel(Context context) + { + try + { + RunProcessInput runProcessInput = new RunProcessInput(); + QJavalinImplementation.setupSession(context, runProcessInput); + + runProcessInput.setProcessName(context.pathParam("processName")); + runProcessInput.setProcessUUID(context.pathParam("processUUID")); + + new CancelProcessAction().execute(runProcessInput); + + Map resultForCaller = new HashMap<>(); + context.result(JsonUtils.toJson(resultForCaller)); + } + catch(Exception e) + { + QJavalinImplementation.handleException(context, e); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java index a352e558..84382e57 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import kong.unirest.HttpResponse; @@ -514,6 +515,42 @@ class QJavalinImplementationTest extends QJavalinTestBase + /******************************************************************************* + ** test an insert that returns a warning + ** + *******************************************************************************/ + @Test + public void test_dataInsertWithWarning() + { + Map body = new HashMap<>(); + body.put("firstName", "Warning"); + body.put("lastName", "Kelkhoff"); + body.put("email", "warning@kelkhoff.com"); + + HttpResponse response = Unirest.post(BASE_URL + "/data/person") + .header("Content-Type", "application/json") + .body(body) + .asString(); + + assertEquals(200, response.getStatus()); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("records")); + JSONArray records = jsonObject.getJSONArray("records"); + assertEquals(1, records.length()); + JSONObject record0 = records.getJSONObject(0); + assertTrue(record0.has("values")); + JSONObject values0 = record0.getJSONObject("values"); + assertTrue(values0.has("id")); + assertEquals(7, values0.getInt("id")); + + assertTrue(record0.has("warnings")); + JSONArray warnings = record0.getJSONArray("warnings"); + assertEquals(1, warnings.length()); + assertTrue(warnings.getJSONObject(0).has("message")); + } + + + /******************************************************************************* ** test an insert - posting a multipart form. ** @@ -594,6 +631,52 @@ class QJavalinImplementationTest extends QJavalinTestBase + /******************************************************************************* + ** test an update - with a warning returned + ** + *******************************************************************************/ + @Test + public void test_dataUpdateWithWarning() + { + Map body = new HashMap<>(); + body.put("firstName", "Warning"); + body.put("birthDate", ""); + + HttpResponse response = Unirest.patch(BASE_URL + "/data/person/4") + .header("Content-Type", "application/json") + .body(body) + .asString(); + + assertEquals(200, response.getStatus()); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("records")); + JSONArray records = jsonObject.getJSONArray("records"); + assertEquals(1, records.length()); + JSONObject record0 = records.getJSONObject(0); + assertTrue(record0.has("values")); + assertEquals("person", record0.getString("tableName")); + JSONObject values0 = record0.getJSONObject("values"); + assertEquals(4, values0.getInt("id")); + assertEquals("Warning", values0.getString("firstName")); + + assertTrue(record0.has("warnings")); + JSONArray warnings = record0.getJSONArray("warnings"); + assertEquals(1, warnings.length()); + assertTrue(warnings.getJSONObject(0).has("message")); + + /////////////////////////////////////////////////////////////////// + // re-GET the record, and validate that birthDate was nulled out // + /////////////////////////////////////////////////////////////////// + response = Unirest.get(BASE_URL + "/data/person/4").asString(); + assertEquals(200, response.getStatus()); + jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("values")); + JSONObject values = jsonObject.getJSONObject("values"); + assertFalse(values.has("birthDate")); + } + + + /******************************************************************************* ** test an update - posting the data as a multipart form ** @@ -761,6 +844,24 @@ class QJavalinImplementationTest extends QJavalinTestBase + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValueWithoutTableOrProcess() + { + HttpResponse response = Unirest.get(BASE_URL + "/possibleValues/person").asString(); + assertEquals(200, response.getStatus()); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertNotNull(jsonObject); + assertNotNull(jsonObject.getJSONArray("options")); + assertEquals(6, jsonObject.getJSONArray("options").length()); + assertEquals(1, jsonObject.getJSONArray("options").getJSONObject(0).getInt("id")); + assertEquals("Darin Kelkhoff (1)", jsonObject.getJSONArray("options").getJSONObject(0).getString("label")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -882,7 +983,7 @@ class QJavalinImplementationTest extends QJavalinTestBase Function makeNewInstanceWithBackendName = (backendName) -> { QInstance newInstance = new QInstance(); - newInstance.addBackend(new QBackendMetaData().withName(backendName).withBackendType("mock")); + newInstance.addBackend(new QBackendMetaData().withName(backendName).withBackendType(MockBackendModule.class)); if(!"invalid".equals(backendName)) { diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java index cd25dee2..74aa407c 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.javalin; import java.net.URLEncoder; import java.nio.charset.Charset; import java.util.List; +import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -575,15 +576,15 @@ class QJavalinProcessHandlerTest extends QJavalinTestBase /******************************************************************************* - ** test calling download file with missing filePath + ** test calling download file without needed query-string params ** *******************************************************************************/ @Test - public void test_downloadFileMissingFilePath() + public void test_downloadFileMissingQueryStringParams() { HttpResponse response = Unirest.get(BASE_URL + "/download/myTestFile.txt").asString(); assertEquals(400, response.getStatus()); - assertTrue(response.getBody().contains("A filePath was not provided")); + assertTrue(response.getBody().contains("Missing query parameters to identify file")); } @@ -604,4 +605,45 @@ class QJavalinProcessHandlerTest extends QJavalinTestBase assertEquals(1, jsonObject.getJSONArray("options").getJSONObject(0).getInt("id")); assertEquals("Darin Kelkhoff (1)", jsonObject.getJSONArray("options").getJSONObject(0).getString("label")); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test_processCancel() + { + ///////////////////////// + // 400s for bad inputs // + ///////////////////////// + { + HttpResponse response = Unirest.get(BASE_URL + "/processes/noSuchProcess/" + UUID.randomUUID() + "/cancel").asString(); + assertEquals(400, response.getStatus()); + assertThat(response.getBody()).contains("Process [noSuchProcess] is not defined in this instance"); + } + { + HttpResponse response = Unirest.get(BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SIMPLE_SLEEP + "/" + UUID.randomUUID() + "/cancel").asString(); + assertEquals(400, response.getStatus()); + assertThat(response.getBody()).matches(".*State for process UUID.*not found.*"); + } + + /////////////////////////////////// + // start a process, get its uuid // + /////////////////////////////////// + String processBasePath = BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SLEEP_INTERACTIVE; + HttpResponse response = Unirest.post(processBasePath + "/init?" + TestUtils.SleeperStep.FIELD_SLEEP_MILLIS + "=" + MORE_THAN_TIMEOUT) + .header("Content-Type", "application/json").asString(); + + JSONObject jsonObject = assertProcessStepCompleteResponse(response); + String processUUID = jsonObject.getString("processUUID"); + assertNotNull(processUUID, "Process UUID should not be null."); + + ///////////////////////////////////////// + // now cancel that, and expect success // + ///////////////////////////////////////// + response = Unirest.get(BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SLEEP_INTERACTIVE + "/" + processUUID + "/cancel").asString(); + assertEquals(200, response.getStatus()); + } + } diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java index 12c1c194..00a87d47 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java @@ -25,6 +25,10 @@ package com.kingsrook.qqq.backend.javalin; import java.io.InputStream; import java.sql.Connection; import java.util.List; +import java.util.Objects; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -35,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -68,6 +73,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.savedviews.SavedViewsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; @@ -225,7 +232,7 @@ public class TestUtils public static QBackendMetaData defineMemoryBackend() { return new QBackendMetaData() - .withBackendType("memory") + .withBackendType(MemoryBackendModule.class) .withName(BACKEND_NAME_MEMORY); } @@ -260,6 +267,9 @@ public class TestUtils .withScriptTypeId(1) .withScriptTester(new QCodeReference(TestScriptAction.class))); + qTableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PersonTableCustomizer.class)); + qTableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(PersonTableCustomizer.class)); + qTableMetaData.getField("photo") .withIsHeavy(true) .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) @@ -271,6 +281,52 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static class PersonTableCustomizer implements TableCustomizerInterface + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + return warnPostInsertOrUpdate(records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + return warnPostInsertOrUpdate(records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List warnPostInsertOrUpdate(List records) + { + for(QRecord record : records) + { + if(Objects.requireNonNullElse(record.getValueString("firstName"), "").toLowerCase().contains("warn")) + { + record.addWarning(new QWarningMessage("Warning in firstName.")); + } + } + + return records; + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java index fe8955eb..2a52e8d7 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java @@ -415,6 +415,7 @@ public class QCommandBuilder { case STRING, TEXT, HTML, PASSWORD -> String.class; case INTEGER -> Integer.class; + case LONG -> Long.class; case DECIMAL -> BigDecimal.class; case DATE -> LocalDate.class; case TIME -> LocalTime.class; diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java index 620433ab..b0583cdb 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java @@ -60,6 +60,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; @@ -618,9 +619,10 @@ public class QPicoCliImplementation ///////////////////////////////////////////// ExportInput exportInput = new ExportInput(); exportInput.setTableName(tableName); - exportInput.setReportFormat(reportFormat); - exportInput.setFilename(filename); - exportInput.setReportOutputStream(outputStream); + exportInput.setReportDestination(new ReportDestination() + .withReportFormat(reportFormat) + .withFilename(filename) + .withReportOutputStream(outputStream)); exportInput.setLimit(subParseResult.matchedOptionValue("limit", null)); exportInput.setQueryFilter(generateQueryFilter(subParseResult)); diff --git a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java index 56ee591c..850491cc 100644 --- a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java +++ b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; @@ -429,8 +430,11 @@ public class QSlackImplementation ExportInput exportInput = new ExportInput(); exportInput.setLimit(1000); exportInput.setTableName(tableName); - exportInput.setReportFormat(ReportFormat.valueOf(format)); - exportInput.setReportOutputStream(baos); + + exportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.valueOf(format)) + .withReportOutputStream(baos)); + setupSession(context, exportInput); ExportOutput output = new ExportAction().execute(exportInput); @@ -662,11 +666,11 @@ public class QSlackImplementation ////////////////////////////////////////////////////////////////////////// // Print result, which includes information about the message (like TS) // ////////////////////////////////////////////////////////////////////////// - LOG.info("Slack post result {}", result); + LOG.info("Slack post result: " + result); } catch(IOException | SlackApiException e) { - LOG.error("error: {}", e.getMessage(), e); + LOG.error("error", e); } } diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index 11c70b15..9d9ea946 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -68,7 +68,7 @@ com.kingsrook.qqq qqq-frontend-material-dashboard - ${revision} + 0.20.0-20240418.180316-42 com.h2database diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java index 4cb6a7cd..d968aaa8 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java @@ -75,7 +75,7 @@ public class ClonePeopleTransformStep extends AbstractTransformStep implements P ** *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { for(QRecord inputPerson : runBackendStepInput.getRecords()) { diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java index 82be6e49..4de7c550 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java @@ -94,7 +94,7 @@ class ClonePeopleTransformStepTest ClonePeopleTransformStep clonePeopleTransformStep = new ClonePeopleTransformStep(); input.setRecords(queryOutput.getRecords()); - clonePeopleTransformStep.run(input, output); + clonePeopleTransformStep.runOnePage(input, output); ArrayList processSummary = clonePeopleTransformStep.getProcessSummary(output, true);