diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..2d19fc76 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +*.html diff --git a/docs/actions/GetAction.adoc b/docs/actions/GetAction.adoc new file mode 100644 index 00000000..697b2513 --- /dev/null +++ b/docs/actions/GetAction.adoc @@ -0,0 +1,64 @@ +== GetAction +include::../variables.adoc[] + +The `*GetAction*` is essentially a subset of the <>, only specifically meant to get just a single record from a {link-table}. +In SQL/RDBMS terms, it is analogous to a `SELECT` statement, where a single record may be found - or - it may not be found. + +For all tables, `GetAction` can do a lookup by primary key. +In addition, for tables that define a `UniqueKey`, name/value pairs (in the form of a `Map`) can be used as input for `GetAction`. + +=== Examples +[source,java] +.Basic form - by primary key +---- +GetInput input = new GetInput(); +input.setTableName("orders"); +input.setPrimaryKey(1); +GetOutput output = new GetAction.execute(input); +QRecord record = output.getRecord(); +---- + +[source,java] +.Secondary form - by Unique Key +---- +GetInput input = new GetInput(); +input.setTableName("products"); +input.setUniqueKey(Map.of("storeId", 1701, "sku", "ABCD")); +GetOutput output = new GetAction.execute(input); +QRecord record = output.getRecord(); +---- + +=== GetInput +* `table` - *String, Required* - Name of the table being queried against. +* `primaryKey` - *Serializable, Conditional* - Value for the primary key field of the table being queried. +Type should match the table's primary key field's type. +If a `primaryKey` is not given, then a `uniqueKey` must be given. +* `uniqueKey` - *Map of String -> Serializable, Conditional* - Map of name-value pairs that define the record to be fetcheed. +Keys in the map must be field names from the table being queried. +Values in the map should should be of types that correspond to the fields. +If a `primaryKey` is not given, then a `uniqueKey` must be given. +If both `primaryKey` and `uniqueKey` are given, then `uniqueKey` is ignored. +* `transaction` - *QBackendTransaction object* - Optional transaction object. +** Behavior for this object is backend-dependant. +In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction. + +* `shouldTranslatePossibleValues` - *boolean, default: false* - Controls whether any fields in the table with a *possibleValueSource* assigned to them should have those possible values looked up +(e.g., to provide text translations in the generated records' `displayValues` map). +** For example, if getting a record to present to a user, this would generally need to be *true*. +But if getting a record as part of a process, then this can generally be left as *false*. +* `shouldGenerateDisplayValues` - *boolean, default: false* - Controls whether field level *displayFormats* should be used to populate the generated records' `displayValues` map. +** For example, if getting a record to present to a user, this would generally need to be *true*. +But if getting a record as part of a process, then this can generally be left as *false*. +* `shouldFetchHeavyFields` - *boolean, default: true* - Controls whether or not fields marked as `isHeavy` should be fetched & returned or not. +* `shouldOmitHiddenFields` - *boolean, default: true* - Controls whether or not fields marked as `isHidden` should be included in the result or not. +* `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned. +* `queryJoins` - *List of <> objects* - Optional list of tables to be joined with the main table being queried. +See QueryJoin under <> for further details. +* `includeAssociations` - *boolean, default: false* - Control whether or not records from tables defined as `associations` under the table being queried should be included in the result or not. +* `associationNamesToInclude` - *Collection of String* - If `includeAssociations` is true, then this field can be used to limit which associated tables are included. +If this field is null, then all associated tables are included. +Otherwise, a table is only included if its name is in this collection. + +=== GetOutput +* `record` - *QRecord* - The record that was specified by the input `primaryKey` or `uniqueKey`. +Will be null if the record was not found. diff --git a/docs/actions/InsertAction.adoc b/docs/actions/InsertAction.adoc new file mode 100644 index 00000000..ee16bafc --- /dev/null +++ b/docs/actions/InsertAction.adoc @@ -0,0 +1,86 @@ +== InsertAction +include::../variables.adoc[] + +To insert (add, create) new records into any {link-table}, the `*InsertAction*` is used. +In SQL/RDBMS terms, it is analogous to a `INSERT` statement, where one or more records can be provided as input. + +=== Examples +[source,java] +.Canonical InsertAction invocation +---- +InsertInput insertInput = new InsertInput(); +insertInput.setTableName("person"); +insertInput.setRecords(personRecordList); +InsertOutput insertOutput = new InsertAction().execute(insertInput); +List insertedPersonRecords = insertOutput.getRecords(); +---- + +=== Details +`InsertAction` does several things beyond just inserting records into the specified table. +A high-level flow of its internal logic is: + +. For tables using an automation status field, set its value to `PENDING_INSERT_AUTOMATIONS` for all `records` that are going to be inserted. +. Perform the following validations, which include running the table's `PRE_INSERT_CUSTOMIZER`, if one is defined, at the time that is specified by the customizer's `WhenToRun` property (default is `AFTER_ALL_VALIDATIONS`): +.. Ensure that default values specified in the table's fields are present if needed. +.. Apply per-field behaviors, as defined in {link-field} meta-data, such as truncating strings that are longer than their specified max. +.. Check for unique key violations (done here instead of in the backend, to provide better error messaging, and to allow a subset of records to be stored while some fail). +_We might want to make an input control in the future to specify that either the full input set should succeed or fail..._ +.. Validate that required fields (again, per {link-field} meta-data) are set, generating per-record errors if they are not. +.. Validate any security fields in the records - e.g., ensure that the user has permission to insert records with the values they are attempting to insert. +. Send the records to the table's backend module to actually insert them into the backend storage. +. If the table has any associations defined, and if associated records are present, then recursively run `InsertAction` on the associated records. +.. In particular, before these recursive `InsertAction` calls are made, values that were generated by the original insert may need to be propagated down into the associated records. +*** For example, if inserting `order` and `lineItem` records, where a {link-join} exists between the two tables on `order.id` and `lineItem.orderId`, and `order.id` values were generated in the first `InsertAction`, then those values are propagated down into the associated `lineItem.orderId` fields. +. If the {link-instance} has an `audit` table, then based on the {link-table}'s audit rules, audits about the inserted records are created. +. If the table has a `POST_INSERT_CUSTOMIZER`, it is executed. + +=== Overloads + +`InsertAction` can be called in a few alternate forms, mostly just for convenience: + +[source,java] +.If inserting a single record, get that record back instead of the InsertOutput: +---- +InsertInput insertInput = new InsertInput(); +insertInput.setTableName("person"); +insertInput.setRecords(List.of(personRecord)); +QRecord insertedRecord = new InsertAction().executeForRecord(insertInput); + +// or more compactly, using InsertInput.withRecord (instead of withRecords) +QRecord insertedRecord = new InsertAction() + .executeForRecord(new InsertInput("person").withRecord(personRecord)); +---- + +[source,java] +.Taking QRecordEntity objects as inputs instead of QRecords: +---- +// insert a list of person entities: +InsertInput insertInput = new InsertInput("person").withRecordEntities(personList); +InsertOutput insertOutput = new InsertAction().execute(insertInput); + +// or for a single person entity (also mapping the output record back to an entity): +Person insertedPerson = new Person(new InsertAction() + .executeForRecord(new InsertInput("person").withRecordEntity(person))); + +---- + +=== InsertInput +* `table` - *String, Required* - Name of the table that records are to be inserted into. +* `records` - *List of QRecord, Required* - List of records to be inserted into the table. +If the list is empty, the insert action does a `noop`. + +.Less common options +* `inputSource` - *InputSource object, default: QInputSource.SYSTEM* - an indicator of what the source of the action is - generally, a `SYSTEM` triggered action, or a `USER` triggered action. +** `InsertAction` will call the `shouldValidateRequiredFields()` method on this object to determine if it should validate that required fields on the records have values. +Both `QInputSource.SYSTEM` and `QInputSource.USER` return `true` from this method, but application can define their own `InputSource` objects with different behavior. +** In addition, this field can be used in pre- and post-insert customizers to drive further custom logic. +* `skipUniqueKeyCheck` - *boolean, default: false* - control whether or not `InsertAction` should check for unique key violations before attempting to insert its records. +** In a context where one has previously done this validation, or is okay letting the backend provide such checks, they may wish to avoid re-doing this work, and thus may set this property to `true`. +* `omitDmlAudit` - *boolean, default: false* - control if the automatic DML audit that `InsertAction` generally performs should be omitted. +* `auditContext` - *String* - optional message which can be included in DML audit messages, to give users more context about why the insert occurred. + + +=== InsertOutput +* `records` - *List of QRecord* - Copy of the input list of records, with details added based on the results of the input action. +** If there were warnings or errors, the corresponding field (`warnings` or `errors`) will be set in the records. +** If the insert action generated any values (such as a serial id or a default value), those values will be in the record's `fields` map. diff --git a/docs/justfile b/docs/justfile new file mode 100644 index 00000000..2d56fe1b --- /dev/null +++ b/docs/justfile @@ -0,0 +1,12 @@ +## https://github.com/casey/just + +default: + just --list + +build-index-html: + asciidoctor -a docinfo=shared index.adoc + line=$(grep 'Last updated' index.html) && sed -i "s/id=\"content\">/&$line/" index.html + +build-and-publish-index-html: build-index-html + scp index.html first-node:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html + @echo "Updated: https://justinsgotskinnylegs.com/qqq-docs.html" diff --git a/docs/metaData/Icons.adoc b/docs/metaData/Icons.adoc new file mode 100644 index 00000000..494b96ff --- /dev/null +++ b/docs/metaData/Icons.adoc @@ -0,0 +1,20 @@ +[#Icons] +== Icons +include::../variables.adoc[] + +#TODO# + +=== QIcon +Icons are defined in a QQQ Instance in a `*QIcon*` object. + +#TODO# + +*QIcon Properties:* + +* `name` - *String* - Name of an icon from the https://mui.com/material-ui/material-icons/[Material UI Icon set] +** Note that icon names from the link above need to be converted from _CamelCase_ to _underscore_case_... +* `path` - *String* - Path to a file served under the application's web root, to be used as the icon. + +_Either `name` or `path` must be specified. If both are given, then name is used._ + + diff --git a/docs/metaData/PermissionRules.adoc b/docs/metaData/PermissionRules.adoc new file mode 100644 index 00000000..fe492dce --- /dev/null +++ b/docs/metaData/PermissionRules.adoc @@ -0,0 +1,13 @@ +[#PermissionRules] +== Permission Rules +include::../variables.adoc[] + +#TODO# + +=== PermissionRule +#TODO# + +*PermissionRule Properties:* + +#TODO# + diff --git a/docs/metaData/QInstance.adoc b/docs/metaData/QInstance.adoc new file mode 100644 index 00000000..882128b1 --- /dev/null +++ b/docs/metaData/QInstance.adoc @@ -0,0 +1,39 @@ +[#QInstance] +== QInstance +include::../variables.adoc[] + +An application in QQQ is defined as a set of Meta Data objects. These objects are all stored together in an object called a `*QInstance*`. + +Currently, a `QInstance` must be programmatically constructed in java code - e.g., by constructing objects which get added to the QInstance, for example: + +[source,java] +.Adding meta-data for two tables and one process to a QInstance +---- +QInstance qInstance = new QInstance(); +qInstance.addTable(definePersonTable()); +qInstance.addTable(defineHomeTable()); +qInstance.addProcess(defineSendPersonHomeProcess()); +---- + +It is on the QQQ roadmap to allow meta-data to be defined in a non-programmatic way, for example, in YAML or JSON files, or even from a dynamic data source (e.g. a document or relational database). + +The middleware and/or frontends being used in an application will drive how the `QInstance` is connected to the running server/process. +For example, using the `qqq-middleware-javalin` module, a the `QJavalinImplementation` class () has a constructor which takes a `QInstance` as an argument: + +[source,java] +.Starting a QQQ Javalin middleware server - passing a QInstance as a parameter to the new QJavalinImplementation +---- +QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); +Javalin service = Javalin.create(); +service.routes(qJavalinImplementation.getRoutes()); +service.start(); +---- + +*QBackendMetaData Setup Methods:* + +These are the methods that one is most likely to use when setting up (defining) a `QInstance` object: + +* asdf + +*QBackendMetaData Usage Methods:* + diff --git a/docs/misc/ProcessBackendSteps.adoc b/docs/misc/ProcessBackendSteps.adoc new file mode 100644 index 00000000..13441f52 --- /dev/null +++ b/docs/misc/ProcessBackendSteps.adoc @@ -0,0 +1,155 @@ +== Process Backend Steps +include::../variables.adoc[] + +In many QQQ applications, much of the code that engineers write will take the form of Backend Steps for {link-processes}. +Such code is defined in classes which implement the interface `BackendStep`. +This interface defines only a single method: +[source,java] +.BackendStep.java +---- +public interface BackendStep +{ + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException; +} + +---- + +Process backend steps have access to state information - specifically, a list of records, and a map of name=value pairs - in the input & output objects. +This state data is persisted by QQQ between steps (e.g., if a frontend step is presented to a user between backend steps). + + +=== RunBackendStepInput +All input data to the step is available in the `RunBackendStepInput` object. +Key methods in this class are: + +* `getRecords()` - Returns the List of <> that are currently being acted on in the process. +* `getValues()` - Returns a Map of String -> Serializable; name=value pairs that are the state-data of the process. +** Values can be added to this state from a process's meta-data, from a screen, or from another backend step. +* `getValue(String fieldName)` - Returns a specific value, by name, from the process's state. +** This method has several variations that return the value as a specific type, such as `getValueString`, `getValueInteger`, `getValueBoolean`... +* `getAsyncJobCallback()` - Accessor for an `AsyncJobCallback` object, which provides a way for the process backend step to communicate about its status or progress with a user running the process in a frontend. +Provides methods: +** `updateStatus(String message)` - Where general status messages can be given. +For example, `"Loading census data"` +** `updateStatus(int current, int total)` - For updating a progress meter. +e.g., "47 of 1701" would be display by calling `.updateStatus(47, 1701)` +* `getFrontendStepBehavior()` - Enum, indicating what should happen when a frontend step is encountered as the process's next step to run. +Possible values are: +** `BREAK` - Indicates that the process's execution should be suspended, so that the screen represented by the frontend step can be presented to a user. +This would be the expected behavior if a process is being run by a user from a UI. +** `SKIP` - Indicates that frontend steps should be skipped. +This would be the expected behavior if a process is running from a scheduled job (without a user present to drive it), for example. +** `FAIL` - Indicates that the process should end with an exception if a frontend step is encountered. +** A backend step may want to act differently based on its frontendStepBehavior. +For example, additional data may be looked up for displaying to a user if the behavior is `BREAK`. +* `getBasepullLastRunTime()` - For <> processes, this is the `Instant` stored in the basepull table as the process's last run time. + +=== RunBackendStepOutput +All output from a process step should be placed in its `RunBackendStepOutput` object (and/or stored to a backend, as appropriate). +Key methods in this class are: + +* `addValue(String fieldName, Serializable value)` - Adds a single named value to the process's state, overwriting it the value if it already exists. +* `addRecord(QRecord record)` - Add a `<>` to the process's output. +* `addAuditSingleInput(AuditSingleInput auditSingleInput)` - Add a new entry to the process's list of audit inputs, to be stored at the completion of the process. +** An `AuditSingleInput` object can most easily be built with the constructor: `AuditSingleInput(String tableName, QRecord record, String auditMessage)`. +** Additional audit details messages (sub-bullets that accompany the primary `auditMessage`) can be added to an `AuditSingleInput` via the `addDetail(String message)` method. +** _Note that at this time, the automatic storing of these audits is only provided by the execute step of a StreamedETLWithFrontendProcesses._ + +=== Example + +[source,java] +.Example of a BackendStep +---- +/******************************************************************************* + ** For the "person" table's "Add Age" process - + ** For each input person record, add the specified yearsToAdd to their age. + *******************************************************************************/ +public class AddAge implements BackendStep +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + ///////////////////////////////////////////////////////////////// + // get the yearsToAdd input field value from the process input // + ///////////////////////////////////////////////////////////////// + Integer yearsToAdd = runBackendStepInput.getValueInteger("yearsToAdd"); + int totalYearsAdded = 0; + + /////////////////////////////////////////////////// + // loop over the records passed into the process // + /////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + Integer age = record.getValueInteger("age"); + age += yearsToAdd; + totalYearsAdded += yearsToAdd; + + //////////////////////////////////////////////////////////////////////////////////////////// + // update the record with the new "age" value. // + // note that this update record object will implicitly be available to the process's next // + // backend step, via the sharing of the processState object. // + //////////////////////////////////////////////////////////////////////////////////////////// + record.setValue("age", age); + } + + ///////////////////////////////////////// + // set an output value for the process // + ///////////////////////////////////////// + runBackendStepOutput.addValue("totalYearsAdded", totalYearsAdded); + } +} + +---- + + +=== Backend Steps for StreamedETLWithFrontendProcesses + +For <> type processes, backend steps are defined a little bit differently than they are for other process types. +In this type of process, the process meta-data defines 3 backend steps which are built-in to QQQ, and which do not have any custom application logic. +These steps are: + +* `StreamedETLPreviewStep` +* `StreamedETLValidateStep` +* `StreamedETLExecuteStep` + +For custom application logic to be implemented in a StreamedETLWithFrontendProcesses, an application engineer must define (up to) 3 backend step classes which are loaded by the steps listed above. +These application-defined steps must extend specific classes (which themselves implement the `BackendStep` interface), to provide the needed logic of this style of process. +These steps are: + +* *Extract* - a subclass of `AbstractExtractStep` - is responsible for Extracting records from the source table. +** For this step, we can often use the QQQ-provided class `ExtractViaQueryStep`, or sometimes a subclass of it. +** The Extract step is called before the Preview, Validate, and Result screens, though for the Preview screen, it is set to only extract a small number of records (10). +* *Transform* - a subclass of `AbstractTransformStep` - is responsible for applying the majority of the business logic of the process. +In ETL terminology, this is the "Transform" action - which means applying some type of logical transformation an input record (found by the Extract step) to generate an output record (stored by the Load step). +** A Transform step's `run` method will be called, potentially, multiple times, each time with a page of records in the `runBackendStepInput` parameter. +** This method is responsible for adding records to the `runBackendStepOutput`, which will then be passed to the *Load* step. +** This class is also responsible for implementing the method `getProcessSummary`, which provides the data to the *Validate* screen. +** The run method will generally update ProcessSummaryLine objects to facilitate this functionality. +** The Transform step is called before the Preview, Validate, and Result screens, consuming all records selected by the Extract step. +* *Load* - a subclass of `AbstractLoadStep` - is responsible for the Load function of the ETL job. +_A quick word on terminology - this step is actually doing what we are more likely to think of as storing data - which feels like the opposite of “loading” - but we use the name Load to keep in line with the ETL naming convention…_ +** The Load step is ONLY called before the Result screen is presented (possibly after Preview, if the user chose to skip validation, otherwise, after validation). +** Similar to the Transform step, the Load step's `run` method will be called potentially multiple times, with pages of records in its input. +** As such, the Load step is generally the only step where data writes should occur. +*** e.g., a Transform step should not do any writes, as it will be called when the user is going to the Preview & Validate screens - e.g., before the user confirmed that they want to execute the action! +** A common pattern is that the Load step just needs to insert or update the list of records output by the Transform step, in which case the QQQ-provided `LoadViaInsertStep` or `LoadViaUpdateStep` can be used, but custom use-cases can be built as well. + +Another distinction between StreamedELTWithFrontendProcess steps and general QQQ process backend steps, is that the list of records in the input & output objects is NOT shared for StreamedELTWithFrontendProcess steps. +The direct implication of this is, that a Transform step MUST explicitly call `output.addRecord()` for any records that it wants to pass along to the Load step. + +==== Example + +[source,java] +.Examples of a Transform and Load step for a StreamedELTWithFrontendProcess +---- + // todo! +---- + +#todo: more details on these 3 specialized types of process steps (e.g., method to overload, when stuff like pre-action is called; how summaries work).# diff --git a/docs/misc/QContext.adoc b/docs/misc/QContext.adoc new file mode 100644 index 00000000..bce19c9c --- /dev/null +++ b/docs/misc/QContext.adoc @@ -0,0 +1,30 @@ +== QContext +include::../variables.adoc[] + +The class `QContext` contains a collection of thread-local variables, to define the current context of the QQQ code that is currently running. +For example, what `QInstance` (meta-data container) is being used, what `QSession` (user attributes) is active, etc. + +Most of the time, main-line application code does not need to worry about setting up the `QContext` - although unit-test code can is a common exception to that rule. +This is because all of QQQ's entry-points into execution (e.g., web context handlers, CLI startup methods, schedulers, and multi-threaded executors) take care of initializing the context. + +It is more common though, for application code to need to get data from the context, such as the current session or any piece of meta-data from the QInstance. +The methods to access data from the `QContext` are fairly straightforward: + +=== Examples +==== Get a QTableMetaData from the active QInstance +[source,java] +---- +QTableMeataData myTable = QContext.getQInstance().getTable("myTable"); +for(QFieldMeataData field : myTable.getFields().values()) +{ + // ... +} +---- + +==== Get a security key value for the current user session +[source,java] +---- +QSession session = QContext.getQSession(); +List clientIds = session.getSecurityKeyValues("clientId"); +---- + diff --git a/docs/misc/QRecordEntities.adoc b/docs/misc/QRecordEntities.adoc new file mode 100644 index 00000000..9fb84841 --- /dev/null +++ b/docs/misc/QRecordEntities.adoc @@ -0,0 +1,116 @@ +== QRecordEntities +include::../variables.adoc[] + +While `<>` provide a flexible mechanism for passing around record data in a QQQ Application, they have one big disadvantage from the point-of-view of a Java Application: +They do not provide a mechanism to ensure compile-time checks of field names or field types. + +As such, an alternative mechanism exists, which allows records in a QQQ application to be worked with following a more familiar Java Bean (Entity Bean) like pattern. +This mechanism is known as a `QRecordEntity`. + +Specifically speaking, `QRecordEntity` is an abstract base class, whose purpose is to be the base class for entity-bean classes. +Using reflection, `QRecordEntity` is able to convert objects back and forth from `QRecord` to specific entity-bean subtypes. + +For example, the method `QRecordEntity::toQRecord()` converts a record entity object into a `QRecord`. + +Inversely, `QRecordEntity::populateFromQRecord(QRecord record)` sets fields in a record entity object, based on the values in the supplied `QRecord`. +It is conventional for a subclass of `QRecordEntity` to have both a no-arg (default) constructor, and a constructor that takes a `QRecord` as a parameter, and calls `populateFromQRecord`. + +In addition to these constructors, a `QRecordEntity` subclass will generally contain: + +* A `public static final String TABLE_NAME`, used throughout the application as a constant reference to the name for the {link-table}. +* A series of `private` fields, corresponding to the fields in the table that the entity represents. +** If these fields are annotated as `@QField()`, then the {link-table} meta-data for the table that the entity represents can in part be inferred by QQQ, by using the method `QTableMetaData::withFieldsFromEntity`. +* `getX()`, `setX()`, and `withX()` methods for all of the entity's fields. + +=== Examples +[source,java] +.Example Definition of a QRecordEntity subclass: Person.java +---- +/******************************************************************************* + ** QRecordEntity for the person table. + *******************************************************************************/ +public class Person extends QRecordEntity +{ + public static final String TABLE_NAME = "person"; + + @QField(isEditable = false) + private Integer id; + + @QField() + private String firstName; + + @QField() + private String lastName; + + @QField() + private Integer age; + + // all other fields + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public Person() + { + } + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public Person(QRecord record) + { + populateFromQRecord(record); + } + + + /******************************************************************************* + ** Custom method to concatenate firstName and lastName + *******************************************************************************/ + public String getFullName() + { + ////////////////////////////////////////////////////////////////////// + // if there were more than an example, we could do some null-checks // + // here to avoid silly output like "Bobby null" :) // + ////////////////////////////////////////////////////////////////////// + return (firstName + " " + lastName); + } + + // all getter, setter, and fluent setter (wither) methods +} +---- + +The core ORM actions and process-execution actions of QQQ work with `QRecords`. +However, application engineers may want to apply patterns like the following example, to gain the compile-time safety of `QRecordEntities`: + +[source,java] +.Example Usage of a QRecordEntity +---- +////////////////////////////////////////////////////////////////////// +// assume we're working with the "person" table & entity from above // +////////////////////////////////////////////////////////////////////// +List recordsToUpdate = new ArrayList<>(); +for(QRecord record : inputRecordList) +{ + ///////////////////////////////////////////////////////////////////////////// + // call the constructor that copies values from the record into the entity // + ///////////////////////////////////////////////////////////////////////////// + Person person = new Person(record); + + //////////////////////////////////////////////// + // call a custom method defined in the entity // + //////////////////////////////////////////////// + LOG.info("Processing: " + person.getFullName()); + + ///////////////////////////////////////////////////////////// + // age is an Integer, so no type-conversion is needed here // + ///////////////////////////////////////////////////////////// + person.setAge(person.getAge() + 1); + + /////////////////////////////////////////////////////////////////////////// + // to pass the updated records to UpdateAction, convert them to QRecords // + /////////////////////////////////////////////////////////////////////////// + recordsToUpdate.add(person.toQRecord()); +} +---- + diff --git a/docs/misc/QRecords.adoc b/docs/misc/QRecords.adoc new file mode 100644 index 00000000..c68df87b --- /dev/null +++ b/docs/misc/QRecords.adoc @@ -0,0 +1,68 @@ +== QRecords +include::../variables.adoc[] + +Almost all code inside a QQQ application will be dealing with *Records* (aka Tuples or Rows). +That is: a collection of named values, representing a single Entity, Fact, or, Row from a {link-table}. + +The class that QQQ uses to work with records is called: `QRecord`. + +=== Values +At its core, a `QRecord` is a wrapper around a `Map values`. +These are the *actual* values for the fields in the table for the record. +That is, direct representations of the values as they are stored in the {link-backend}. + +The keys in the `values` map are names from the {link-fields} in the {link-table}. + +The values in `values` map are declared as `Serializable` (to help ensure the serializability of the `QRecord` as a whole). +In practice, their types will be based on the `QFieldType` of the {link-field} that they correspond to. +That will typically be one of: `String`, `Integer`, `Boolean`, `BigDecimal`, `Instant`, `LocalDate`, `LocalTime`, or `byte[]`. +Be aware that `null` values may be in the `values` map, especially/per if the backend/table support `null`. + +To work with the `values` map, the following methods are provided: + +* `setValue(String fieldName, Serializable value)` - Sets a value for the specified field in the record. +** Overloaded as `setValue(String fieldName, Object value)` - For some cases where the value may not be known to be `Serializable`. +In this overload, if the value is `null` or `Serializable`, the primary version of `setValue` is called. +Otherwise, the `value` is passed through `String::valueOf`, and the result is stored. +** Overloaded as `setValue(QFieldMetaData field, Serializable value)` - which simply defers to the primary version of `setValue`, passing `field.getName()` as the first parameter. +* `removeValue(String fieldName)` - Remove the given field from the `values` map. +** Note that in some situations this is an important distinction from having a `null` value in the map (See <>). +* `setValues(Map values)` - Sets the full map of `values`. +* `getValues()` - Returns the full map of `values`. +* `getValue(String fieldName)` - Returns the value for the named field - possibly `null` - as a `Serializable`. +* Several type-specific variations of `getValueX(String fieldName)`, where internally, values will be not exactly type-cast, but effectively converted (if possible) to the requested type. +These conversions are done using the `ValueUtils.getValueAsX(Object)` methods. +These methods are generally the preferred/cleanest way to get record values in application code, when it is needed in a type-specific way . +** `getValueString(String fieldName)` +** `getValueInteger(String fieldName)` +** `getValueBoolean(String fieldName)` +** `getValueBigDecimal(String fieldName)` +** `getValueInstant(String fieldName)` +** `getValueLocalDate(String fieldName)` +** `getValueLocalTime(String fieldName)` +** `getValueByteArray(String fieldName)` + +=== Display Values +In addition to the `values` map, a `QRecord` contains another map called `displayValues`, which only stores `String` values. +That is to say, values for other types are stringified, based on their {link-field}'s type and `displayFormat` property. +In addition, fields which have a `possibleValueSource` property will have their translated values set in the `displayValues` map. + +By default, a `QRecord` will not have its `displayValues` populated. +To populate `displayValues`, the <> and <> classes take a property in their inputs called `shouldGenerateDisplayValues`, which must be set to `true` to generate `displayValues`. +In addition, these two actions also have a property `shouldTranslatePossibleValues` in their inputs, which needs to be set to `true` if possible-value lookups are to be performed. + +As an alternative to the `shouldGenerateDisplayValues` and `shouldTranslatePossibleValues` inputs to <> and <>, one can directly call the `QValueFormatter.setDisplayValuesInRecords` and/or `qPossibleValueTranslator.translatePossibleValuesInRecords` methods. +Or, for special cases, `setDisplayValue(String fieldName, String displayValue)` or `setDisplayValues(Map displayValues)` can be called directly. + +=== Backend Details +Sometimes a backend may want to place additional data in a `QRecord` that doesn't exactly correspond to a field. +To do this, the `Map backendDetails` member is used. + +For example, an API backend may store the full JSON `String` that came from the API as a backend detail in a `QRecord`. +Or fields that are marked as `isHeavy`, if the full (heavy) value of the field hasn't been fetched, then the lengths of any such heavy fields may be stored in `backendDetails`. + +=== Errors and Warnings +#todo# + +=== Associated Records +#todo# diff --git a/pom.xml b/pom.xml index ade15061..79723366 100644 --- a/pom.xml +++ b/pom.xml @@ -330,28 +330,28 @@ fi - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.6.2 - - - aggregate - false - - aggregate - - - - default - - javadoc - - - - - + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.6.2 + + + aggregate + false + + aggregate + + + + default + + javadoc + + + + + diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 4582bbec..8da52458 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.instances; import java.io.Serializable; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.text.ParseException; import java.util.ArrayList; @@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; @@ -55,9 +57,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -67,16 +71,21 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; @@ -93,6 +102,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase; import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; @@ -115,6 +125,8 @@ public class QInstanceValidator private boolean printWarnings = false; + private static ListingHash, QInstanceValidatorPluginInterface> validatorPlugins = new ListingHash<>(); + private List errors = new ArrayList<>(); @@ -175,6 +187,8 @@ public class QInstanceValidator validateSupplementalMetaData(qInstance); validateUniqueTopLevelNames(qInstance); + + runPlugins(QInstance.class, qInstance, qInstance); } catch(Exception e) { @@ -193,6 +207,57 @@ public class QInstanceValidator + /******************************************************************************* + ** + *******************************************************************************/ + public static void addValidatorPlugin(QInstanceValidatorPluginInterface plugin) + { + Optional validateMethod = Arrays.stream(plugin.getClass().getDeclaredMethods()) + .filter(m -> m.getName().equals("validate") + && m.getParameterCount() == 3 + && !m.getParameterTypes()[0].equals(Object.class) + && m.getParameterTypes()[1].equals(QInstance.class) + && m.getParameterTypes()[2].equals(QInstanceValidator.class) + ).findFirst(); + + if(validateMethod.isPresent()) + { + Class parameterType = validateMethod.get().getParameterTypes()[0]; + validatorPlugins.add(parameterType, plugin); + } + else + { + LOG.warn("Could not find validate method on validator plugin [" + plugin.getClass().getName() + "] (to infer type being validated) - this plugin will not be used."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void removeAllValidatorPlugins() + { + validatorPlugins.clear(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runPlugins(Class c, T t, QInstance qInstance) + { + for(QInstanceValidatorPluginInterface plugin : CollectionUtils.nonNullList(validatorPlugins.get(c))) + { + @SuppressWarnings("unchecked") + QInstanceValidatorPluginInterface processPlugin = (QInstanceValidatorPluginInterface) plugin; + processPlugin.validate(t, qInstance, this); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -201,6 +266,8 @@ public class QInstanceValidator for(QSupplementalInstanceMetaData supplementalInstanceMetaData : CollectionUtils.nonNullMap(qInstance.getSupplementalMetaData()).values()) { supplementalInstanceMetaData.validate(qInstance, this); + + runPlugins(QSupplementalInstanceMetaData.class, supplementalInstanceMetaData, qInstance); } } @@ -238,6 +305,8 @@ public class QInstanceValidator { assertCondition(qInstance.getPossibleValueSource(securityKeyType.getPossibleValueSourceName()) != null, "Unrecognized possibleValueSourceName in securityKeyType: " + name); } + + runPlugins(QSecurityKeyType.class, securityKeyType, qInstance); } }); } @@ -284,6 +353,8 @@ public class QInstanceValidator assertNoException(() -> qInstance.getTable(join.getRightTable()).getField(orderBy.getFieldName()), "Field name " + orderBy.getFieldName() + " in orderBy for join " + joinName + " is not a defined field in the right-table " + join.getRightTable()); } } + + runPlugins(QJoinMetaData.class, join, qInstance); }); } @@ -352,6 +423,8 @@ public class QInstanceValidator validateScheduleMetaData(sqsQueueProvider.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: "); } } + + runPlugins(QQueueProviderMetaData.class, queueProvider, qInstance); }); } @@ -366,6 +439,8 @@ public class QInstanceValidator { assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name); } + + runPlugins(QQueueMetaData.class, queue, qInstance); }); } } @@ -384,6 +459,8 @@ public class QInstanceValidator assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); backend.performValidation(this); + + runPlugins(QBackendMetaData.class, backend, qInstance); }); } } @@ -406,6 +483,8 @@ public class QInstanceValidator { validateScheduleMetaData(automationProvider.getSchedule(), qInstance, "automationProvider " + name + ", schedule: "); } + + runPlugins(QAutomationProviderMetaData.class, automationProvider, qInstance); }); } } @@ -424,6 +503,8 @@ public class QInstanceValidator { validateSimpleCodeReference("Instance Authentication meta data customizer ", authentication.getCustomizer(), QAuthenticationModuleCustomizerInterface.class); } + + runPlugins(QAuthenticationMetaData.class, authentication, qInstance); } } @@ -546,6 +627,8 @@ public class QInstanceValidator { supplementalTableMetaData.validate(qInstance, table, this); } + + runPlugins(QTableMetaData.class, table, qInstance); }); } } @@ -1344,6 +1427,7 @@ public class QInstanceValidator supplementalProcessMetaData.validate(qInstance, process, this); } + runPlugins(QProcessMetaData.class, process, qInstance); }); } } @@ -1494,6 +1578,8 @@ public class QInstanceValidator // view.getTitleFormat(); view.getTitleFields(); // validate these match? } } + + runPlugins(QReportMetaData.class, report, qInstance); }); } } @@ -1614,9 +1700,9 @@ public class QInstanceValidator } } - ////////////////////////////////////////// - // validate field sections in the table // - ////////////////////////////////////////// + //////////////////////////////////////// + // validate field sections in the app // + //////////////////////////////////////// Set childNamesInSections = new HashSet<>(); if(app.getSections() != null) { @@ -1644,6 +1730,8 @@ public class QInstanceValidator { assertCondition(qInstance.getWidget(widgetName) != null, "App " + appName + " widget " + widgetName + " is not a recognized widget."); } + + runPlugins(QAppMetaData.class, app, qInstance); }); } } @@ -1676,6 +1764,8 @@ public class QInstanceValidator } } } + + runPlugins(QWidgetMetaDataInterface.class, widget, qInstance); } ); } @@ -1761,6 +1851,8 @@ public class QInstanceValidator } default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType()); } + + runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance); } }); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidator.java new file mode 100644 index 00000000..1ca46415 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidator.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.validation.plugins; + + +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullExtractStepInterface; + + +/******************************************************************************* + ** instance validator plugin, to ensure that a process which is a basepull uses + ** an extract step marked for basepulls. + *******************************************************************************/ +public class BasepullExtractStepValidator implements QInstanceValidatorPluginInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validate(QProcessMetaData process, QInstance qInstance, QInstanceValidator qInstanceValidator) + { + /////////////////////////////////////////////////////////////////////////// + // if there's no basepull config on the process, don't do any validation // + /////////////////////////////////////////////////////////////////////////// + if(process.getBasepullConfiguration() == null) + { + return; + } + + ////////////////////////////////////////////////////////////////////////////////////////// + // try to find an input field in the process, w/ a defaultValue that's a QCodeReference // + // and is an instance of BasepullExtractStepInterface // + ////////////////////////////////////////////////////////////////////////////////////////// + boolean foundBasepullExtractStep = false; + for(QFieldMetaData field : process.getInputFields()) + { + if(field.getDefaultValue() != null && field.getDefaultValue() instanceof QCodeReference codeReference) + { + try + { + BasepullExtractStepInterface extractStep = QCodeLoader.getAdHoc(BasepullExtractStepInterface.class, codeReference); + if(extractStep != null) + { + foundBasepullExtractStep = true; + } + } + catch(Exception e) + { + ////////////////////////////////////////////////////// + // ok, just means we haven't found our extract step // + ////////////////////////////////////////////////////// + } + } + } + + /////////////////////////////////////////////////////////// + // validate we could find a BasepullExtractStepInterface // + /////////////////////////////////////////////////////////// + qInstanceValidator.assertCondition(foundBasepullExtractStep, "Process [" + process.getName() + "] has a basepullConfiguration, but does not have a field with a default value that is a BasepullExtractStepInterface CodeReference"); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceValidatorPluginInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceValidatorPluginInterface.java new file mode 100644 index 00000000..9123d398 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceValidatorPluginInterface.java @@ -0,0 +1,41 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.validation.plugins; + + +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; + + +/******************************************************************************* + ** Interface for additional / optional q instance validators. Some will be + ** provided by QQQ - others can be defined by applications. + *******************************************************************************/ +public interface QInstanceValidatorPluginInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void validate(T object, QInstance qInstance, QInstanceValidator qInstanceValidator); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java new file mode 100644 index 00000000..ad96eac5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java @@ -0,0 +1,153 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.logging; + + +import org.apache.logging.log4j.Level; +import org.json.JSONException; +import org.json.JSONObject; + + +/******************************************************************************* + ** A log message, which can be "collected" by the QCollectingLogger. + *******************************************************************************/ +public class CollectedLogMessage +{ + private Level level; + private String message; + private Throwable exception; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public CollectedLogMessage() + { + } + + + + /******************************************************************************* + ** Getter for message + *******************************************************************************/ + public String getMessage() + { + return (this.message); + } + + + + /******************************************************************************* + ** Setter for message + *******************************************************************************/ + public void setMessage(String message) + { + this.message = message; + } + + + + /******************************************************************************* + ** Fluent setter for message + *******************************************************************************/ + public CollectedLogMessage withMessage(String message) + { + this.message = message; + return (this); + } + + + + /******************************************************************************* + ** Getter for exception + *******************************************************************************/ + public Throwable getException() + { + return (this.exception); + } + + + + /******************************************************************************* + ** Setter for exception + *******************************************************************************/ + public void setException(Throwable exception) + { + this.exception = exception; + } + + + + /******************************************************************************* + ** Fluent setter for exception + *******************************************************************************/ + public CollectedLogMessage withException(Throwable exception) + { + this.exception = exception; + return (this); + } + + + + /******************************************************************************* + ** Getter for level + ** + *******************************************************************************/ + public Level getLevel() + { + return level; + } + + + + /******************************************************************************* + ** Setter for level + ** + *******************************************************************************/ + public void setLevel(Level level) + { + this.level = level; + } + + + + /******************************************************************************* + ** Fluent setter for level + ** + *******************************************************************************/ + public CollectedLogMessage withLevel(Level level) + { + this.level = level; + return (this); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public JSONObject getMessageAsJSONObject() throws JSONException + { + return (new JSONObject(getMessage())); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java new file mode 100644 index 00000000..b10d3c07 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java @@ -0,0 +1,155 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.logging; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ObjectMessage; +import org.apache.logging.log4j.message.SimpleMessageFactory; +import org.apache.logging.log4j.simple.SimpleLogger; +import org.apache.logging.log4j.util.PropertiesUtil; + + +/******************************************************************************* + ** QQQ log4j implementation, used within a QLogger, to "collect" log messages + ** in an internal list - the idea being - for tests, to assert that logs happened. + *******************************************************************************/ +public class QCollectingLogger extends SimpleLogger +{ + private List collectedMessages = new ArrayList<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // just in case one of these gets activated, and left on, put a limit on how many messages we'll collect // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + private int capacity = 100; + + private Logger logger; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QCollectingLogger(Logger logger) + { + super(logger.getName(), logger.getLevel(), false, false, true, false, "", new SimpleMessageFactory(), new PropertiesUtil(new Properties()), System.out); + this.logger = logger; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void logMessage(String fqcn, Level level, Marker marker, Message message, Throwable throwable) + { + //////////////////////////////////////////// + // add this log message to our collection // + //////////////////////////////////////////// + collectedMessages.add(new CollectedLogMessage() + .withLevel(level) + .withMessage(message.getFormattedMessage()) + .withException(throwable)); + + //////////////////////////////////////////////////////////////////////////////////////// + // if we've gone over our capacity, remove the 1st entry until we're back at capacity // + //////////////////////////////////////////////////////////////////////////////////////// + while(collectedMessages.size() > capacity) + { + collectedMessages.remove(0); + } + + ////////////////////////////////////////////////////////////////////// + // update the message that we log to indicate that we collected it. // + // if it looks like JSON, insert as a name:value pair; else text. // + ////////////////////////////////////////////////////////////////////// + String formattedMessage = message.getFormattedMessage(); + String updatedMessage; + if(formattedMessage.startsWith("{")) + { + updatedMessage = """ + {"collected":true,""" + formattedMessage.substring(1); + } + else + { + updatedMessage = "[Collected] " + formattedMessage; + } + ObjectMessage myMessage = new ObjectMessage(updatedMessage); + + /////////////////////////////////////////////////////////////////////////////////////// + // log the message with the original log4j logger, with our slightly updated message // + /////////////////////////////////////////////////////////////////////////////////////// + logger.logMessage(level, marker, fqcn, null, myMessage, throwable); + } + + + + /******************************************************************************* + ** Setter for logger + ** + *******************************************************************************/ + public void setLogger(Logger logger) + { + this.logger = logger; + } + + + + /******************************************************************************* + ** Getter for collectedMessages + ** + *******************************************************************************/ + public List getCollectedMessages() + { + return collectedMessages; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void clear() + { + this.collectedMessages.clear(); + } + + + + /******************************************************************************* + ** Setter for capacity + ** + *******************************************************************************/ + public void setCapacity(int capacity) + { + this.capacity = capacity; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java index 3683c62a..a00dfb0d 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java @@ -119,6 +119,34 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public static QCollectingLogger activateCollectingLoggerForClass(Class c) + { + Logger loggerFromLogManager = LogManager.getLogger(c); + QCollectingLogger collectingLogger = new QCollectingLogger(loggerFromLogManager); + + QLogger qLogger = getLogger(c); + qLogger.setLogger(collectingLogger); + + return collectingLogger; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void deactivateCollectingLoggerForClass(Class c) + { + Logger loggerFromLogManager = LogManager.getLogger(c); + QLogger qLogger = getLogger(c); + qLogger.setLogger(loggerFromLogManager); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -518,7 +546,7 @@ public class QLogger /******************************************************************************* ** *******************************************************************************/ - private String makeJsonString(String message, Throwable t, List logPairList) + protected String makeJsonString(String message, Throwable t, List logPairList) { if(logPairList == null) { @@ -620,4 +648,15 @@ public class QLogger exceptionList.get(0).setHasLoggedLevel(level); return (level); } + + + + /******************************************************************************* + ** Setter for logger + ** + *******************************************************************************/ + private void setLogger(Logger logger) + { + this.logger = logger; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java index 59a46c69..84b3ab31 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java @@ -287,16 +287,22 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, ////////////////////////////////////////////////////////////////////////////////////// if(!recordsToUpdate.isEmpty() && !isReview) { - LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size())); + LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("countByStatus", countByStatus)); new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true)); } - for(Map.Entry entry : countByStatus.entrySet()) + /////////////////////////////////////////////////// + // on the review step, add records to the output // + /////////////////////////////////////////////////// + if(isReview) { - runBackendStepOutput.addRecord(new QRecord() - .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel()) - .withValue("badStatus", entry.getKey()) - .withValue("count", entry.getValue())); + for(Map.Entry entry : countByStatus.entrySet()) + { + runBackendStepOutput.addRecord(new QRecord() + .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel()) + .withValue("badStatus", entry.getKey()) + .withValue("count", entry.getValue())); + } } return (recordsToUpdate.size()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullExtractStepInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullExtractStepInterface.java new file mode 100644 index 00000000..d448a9fd --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullExtractStepInterface.java @@ -0,0 +1,30 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.basepull; + + +/******************************************************************************* + ** marker - used to indicate that a step is meant to be used for basepull extracts. + *******************************************************************************/ +public interface BasepullExtractStepInterface +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java index 1bc357ea..88e3352c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java @@ -38,7 +38,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith /******************************************************************************* ** Version of ExtractViaQueryStep that knows how to set up a basepull query. *******************************************************************************/ -public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep +public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements BasepullExtractStepInterface { /******************************************************************************* diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 251d42f7..30b3a319 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.AlwaysFailsProcessValidatorPlugin; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; @@ -99,7 +100,7 @@ import static org.junit.jupiter.api.Assertions.fail; ** Unit test for QInstanceValidator. ** *******************************************************************************/ -class QInstanceValidatorTest extends BaseTest +public class QInstanceValidatorTest extends BaseTest { /******************************************************************************* @@ -490,6 +491,35 @@ class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test_validatorPlugins() + { + try + { + QInstanceValidator.addValidatorPlugin(new AlwaysFailsProcessValidatorPlugin()); + + //////////////////////////////////////// + // make sure our always failer fails. // + //////////////////////////////////////// + assertValidationFailureReasonsAllowingExtraReasons((qInstance) -> { + }, "always fail"); + } + finally + { + QInstanceValidator.removeAllValidatorPlugins(); + + //////////////////////////////////////////////////// + // make sure if remove all plugins, we don't fail // + //////////////////////////////////////////////////// + assertValidationSuccess((qInstance) -> {}); + } + } + + + /******************************************************************************* ** Test that a table with no fields fails. ** @@ -2072,9 +2102,9 @@ class QInstanceValidatorTest extends BaseTest ** failed validation with reasons that match the supplied vararg-reasons (but allow ** more reasons - e.g., helpful when one thing we're testing causes other errors). *******************************************************************************/ - private void assertValidationFailureReasonsAllowingExtraReasons(Consumer setup, String... reasons) + public static void assertValidationFailureReasonsAllowingExtraReasons(Consumer setup, String... expectedReasons) { - assertValidationFailureReasons(setup, true, reasons); + assertValidationFailureReasons(setup, true, expectedReasons); } @@ -2084,9 +2114,9 @@ class QInstanceValidatorTest extends BaseTest ** failed validation with reasons that match the supplied vararg-reasons (and ** require that exact # of reasons). *******************************************************************************/ - private void assertValidationFailureReasons(Consumer setup, String... reasons) + public static void assertValidationFailureReasons(Consumer setup, String... expectedReasons) { - assertValidationFailureReasons(setup, false, reasons); + assertValidationFailureReasons(setup, false, expectedReasons); } @@ -2094,7 +2124,7 @@ class QInstanceValidatorTest extends BaseTest /******************************************************************************* ** Implementation for the overloads of this name. *******************************************************************************/ - private void assertValidationFailureReasons(Consumer setup, boolean allowExtraReasons, String... reasons) + public static void assertValidationFailureReasons(Consumer setup, boolean allowExtraReasons, String... expectedReasons) { try { @@ -2105,17 +2135,27 @@ class QInstanceValidatorTest extends BaseTest } catch(QInstanceValidationException e) { - if(!allowExtraReasons) - { - int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size(); - assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons) - + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--")); - } + assertValidationFailureReasons(allowExtraReasons, e.getReasons(), expectedReasons); + } + } - for(String reason : reasons) - { - assertReason(reason, e); - } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void assertValidationFailureReasons(boolean allowExtraReasons, List actualReasons, String... expectedReasons) + { + if(!allowExtraReasons) + { + int noOfReasons = actualReasons == null ? 0 : actualReasons.size(); + assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons) + + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); + } + + for(String reason : expectedReasons) + { + assertReason(reason, actualReasons); } } @@ -2145,11 +2185,11 @@ class QInstanceValidatorTest extends BaseTest ** the list of reasons in the QInstanceValidationException. ** *******************************************************************************/ - private void assertReason(String reason, QInstanceValidationException e) + public static void assertReason(String reason, List actualReasons) { - assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)"); - assertThat(e.getReasons()) - .withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason) + assertNotNull(actualReasons, "Expected there to be a reason for the failure (but there was not)"); + assertThat(actualReasons) + .withFailMessage("Expected any of:\n%s\nTo match: [%s]", actualReasons, reason) .anyMatch(s -> s.contains(reason)); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/AlwaysFailsProcessValidatorPlugin.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/AlwaysFailsProcessValidatorPlugin.java new file mode 100644 index 00000000..8a26a130 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/AlwaysFailsProcessValidatorPlugin.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.validation.plugins; + + +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; + + +/******************************************************************************* + ** test instance of a validator plugin, that always adds an error + *******************************************************************************/ +public class AlwaysFailsProcessValidatorPlugin implements QInstanceValidatorPluginInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validate(QProcessMetaData object, QInstance qInstance, QInstanceValidator qInstanceValidator) + { + qInstanceValidator.getErrors().add("I always fail."); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidatorTest.java new file mode 100644 index 00000000..cf0516a0 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidatorTest.java @@ -0,0 +1,109 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.validation.plugins; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; +import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcessTest; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.instances.QInstanceValidatorTest.assertValidationFailureReasons; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for BasepullExtractStepValidator + *******************************************************************************/ +class BasepullExtractStepValidatorTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNoExtractStepAtAllFails() + { + QInstance qInstance = QContext.getQInstance(); + QInstanceValidator validator = new QInstanceValidator(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // turns out, our "basepullTestProcess" doesn't have an extract step, so that makes this condition fire. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + new BasepullExtractStepValidator().validate(qInstance.getProcess(TestUtils.PROCESS_NAME_BASEPULL), qInstance, validator); + assertValidationFailureReasons(false, validator.getErrors(), "does not have a field with a default value that is a BasepullExtractStepInterface CodeReference"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExtractViaQueryNotBasePull() + { + QInstance qInstance = QContext.getQInstance(); + QInstanceValidator validator = new QInstanceValidator(); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up a streamed-etl process, but with an ExtractViaQueryStep instead of a basepull - it should fail! // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + new BasepullExtractStepValidator().validate(StreamedETLWithFrontendProcess.defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_PERSON_MEMORY, + ExtractViaQueryStep.class, + StreamedETLWithFrontendProcessTest.TestTransformShapeToPersonStep.class, + LoadViaInsertStep.class).withBasepullConfiguration(new BasepullConfiguration()), qInstance, validator); + assertValidationFailureReasons(false, validator.getErrors(), "does not have a field with a default value that is a BasepullExtractStepInterface CodeReference"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExtractViaBasepullQueryPasses() + { + QInstance qInstance = QContext.getQInstance(); + QInstanceValidator validator = new QInstanceValidator(); + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // set up a streamed-etl process, with an ExtractViaBasepullQueryStep as expected - should pass // + ////////////////////////////////////////////////////////////////////////////////////////////////// + new BasepullExtractStepValidator().validate(StreamedETLWithFrontendProcess.defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_PERSON_MEMORY, + ExtractViaBasepullQueryStep.class, + StreamedETLWithFrontendProcessTest.TestTransformShapeToPersonStep.class, + LoadViaInsertStep.class).withBasepullConfiguration(new BasepullConfiguration()), qInstance, validator); + assertThat(validator.getErrors()).isNullOrEmpty(); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QCollectingLoggerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QCollectingLoggerTest.java new file mode 100644 index 00000000..8a70203e --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QCollectingLoggerTest.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.logging; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.apache.logging.log4j.Level; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for QCollectingLogger + *******************************************************************************/ +class QCollectingLoggerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + ClassThatLogsThings classThatLogsThings = new ClassThatLogsThings(); + classThatLogsThings.logAnInfo("1"); + + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(ClassThatLogsThings.class); + classThatLogsThings.logAnInfo("2"); + classThatLogsThings.logAWarn("3"); + QLogger.deactivateCollectingLoggerForClass(ClassThatLogsThings.class); + + classThatLogsThings.logAWarn("4"); + + assertEquals(2, collectingLogger.getCollectedMessages().size()); + + assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains(""" + "message":"2","""); + assertEquals("2", collectingLogger.getCollectedMessages().get(0).getMessageAsJSONObject().getString("message")); + assertEquals(Level.INFO, collectingLogger.getCollectedMessages().get(0).getLevel()); + + assertThat(collectingLogger.getCollectedMessages().get(1).getMessage()).contains(""" + "message":"3","""); + assertEquals(Level.WARN, collectingLogger.getCollectedMessages().get(1).getLevel()); + assertEquals("3", collectingLogger.getCollectedMessages().get(1).getMessageAsJSONObject().getString("message")); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class ClassThatLogsThings + { + private static final QLogger LOG = QLogger.getLogger(ClassThatLogsThings.class); + + /******************************************************************************* + ** + *******************************************************************************/ + private void logAnInfo(String message) + { + LOG.info(message); + } + + /******************************************************************************* + ** + *******************************************************************************/ + private void logAWarn(String message) + { + LOG.warn(message); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java index caf74762..5214c5c7 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java @@ -29,13 +29,20 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; @@ -74,6 +81,43 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest + /******************************************************************************* + ** at one point, when the review step go added, we were double-adding records + ** to the output/result screen. This test verifies, if we run the full process + ** that that doesn't happen. + *******************************************************************************/ + @Test + void testTwoFailedUpdatesFullProcess() throws QException + { + QContext.getQInstance().addProcess(new HealBadRecordAutomationStatusesProcessStep().produce(QContext.getQInstance())); + + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(new QRecord(), new QRecord()))); + List records = queryAllRecords(); + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, null); + + assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r))); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(HealBadRecordAutomationStatusesProcessStep.NAME); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, records.stream().map(r -> r.getValue("id")).toList())))); + RunProcessAction runProcessAction = new RunProcessAction(); + RunProcessOutput runProcessOutput = runProcessAction.execute(input); + + input.setStartAfterStep(runProcessOutput.getProcessState().getNextStepName().get()); + runProcessOutput = runProcessAction.execute(input); + + input.setStartAfterStep(runProcessOutput.getProcessState().getNextStepName().get()); + runProcessOutput = runProcessAction.execute(input); + + List outputRecords = runProcessOutput.getProcessState().getRecords(); + assertEquals(1, outputRecords.size()); + assertEquals(2, outputRecords.get(0).getValueInteger("count")); + + assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r))); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-dev-tools/bin/end-of-sprint-release.sh b/qqq-dev-tools/bin/end-of-sprint-release.sh index 9b6e701f..6847af1a 100755 --- a/qqq-dev-tools/bin/end-of-sprint-release.sh +++ b/qqq-dev-tools/bin/end-of-sprint-release.sh @@ -60,7 +60,7 @@ cd $QQQ_RELEASE_DIR/qqq || exit MVN_VERIFY_LOG=/tmp/mvn-verify.log gumBanner "Doing clean build (logging to $MVN_VERIFY_LOG)" -mvn clean verify > $MVN_VERIFY_LOG 2>&1 +mvn -Duser.timezone=UTC clean verify > $MVN_VERIFY_LOG 2>&1 tail -30 $MVN_VERIFY_LOG gumConfirmProceed "Can we Proceed, or are there build errors to fix?" "Proceed" "There are build errors to fix" diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index e41bd0ea..11c70b15 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -68,7 +68,7 @@ com.kingsrook.qqq qqq-frontend-material-dashboard - feature-CE-876-develop-missing-widget-types-20240221.011827-1 + ${revision} com.h2database