diff --git a/.circleci/config.yml b/.circleci/config.yml index b7346832..7433d19e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2.1 orbs: localstack: localstack/platform@2.1 + browser-tools: circleci/browser-tools@1.4.7 commands: store_jacoco_site: @@ -38,6 +39,8 @@ commands: - restore_cache: keys: - v1-dependencies-{{ checksum "pom.xml" }} + - browser-tools/install-chrome + - browser-tools/install-chromedriver - run: name: Write .env command: | diff --git a/docs/actions/QueryAction.adoc b/docs/actions/QueryAction.adoc index dfff3655..8e5de1f6 100644 --- a/docs/actions/QueryAction.adoc +++ b/docs/actions/QueryAction.adoc @@ -155,9 +155,9 @@ new QFilterOrderBy() ---- ==== QueryJoin -* `joinTable` - *String, required* - Name of the table that is being joined in to the existing query. +* `joinTable` - *String, required (though inferrable)* - Name of the table that is being joined in to the existing query. ** Will be inferred from *joinMetaData*, if *joinTable* is not set when *joinMetaData* gets set. -* `baseTableOrAlias` - *String, required* - Name of a table (or an alias) already defined in the query, to which the *joinTable* will be joined. +* `baseTableOrAlias` - *String, required (though inferrable)* - Name of a table (or an alias) already defined in the query, to which the *joinTable* will be joined. ** Will be inferred from *joinMetaData*, if *baseTableOrAlias* is not set when *joinMetaData* gets set (which will only use the leftTableName from the joinMetaData - never an alias). * `joinMetaData` - *QJoinMetaData object* - Optional specification of a {link-join} in the current QInstance. If not set, will be looked up at runtime based on *baseTableOrAlias* and *joinTable*. @@ -165,21 +165,78 @@ If not set, will be looked up at runtime based on *baseTableOrAlias* and *joinTa * `alias` - *String* - Optional (unless multiple instances of the same table are being joined together, when it becomes required). Behavior based on SQL `FROM` clause aliases. If given, must be used as the part before the dot in field name specifications throughout the rest of the query input. -* `select` - *boolean, default: false* - Specify whether fields from the *rightTable* should be selected by the query. +* `select` - *boolean, default: false* - Specify whether fields from the *joinTable* should be selected by the query. If *true*, then the `QRecord` objects returned by this query will have values with corresponding to the (table-or-alias `.` field-name) form. * `type` - *Enum of INNER, LEFT, RIGHT, FULL, default: INNER* - specifies the SQL-style type of join being performed. [source,java] -.QueryJoin definition examples: +.Basic QueryJoin usage example: ---- -// selecting from an "orderLine" table - then join to its corresponding "order" table +// selecting from an "orderLine" table, joined to its corresponding (parent) "order" table queryInput.withTableName("orderLine"); queryInput.withQueryJoin(new QueryJoin("order").withSelect(true)); ... queryOutput.getRecords().get(0).getValueBigDecimal("order.grandTotal"); +---- +[source,java] +."V" shaped query - selecting from one parent table, and two children joined to it: +---- +// TODO this needs verified for accuracy, though is a reasonable starting point as-is +// selecting from an "order" table, and two children of it, orderLine and customer +queryInput.withTableName("order"); +queryInput.withQueryJoin(new QueryJoin("orderLine").withSelect(true)); +queryInput.withQueryJoin(new QueryJoin("customer").withSelect(true)); +... +QRecord joinedRecord = queryOutput.getRecords().get(0); +joinedRecord.getValueString("orderNo"); +joinedRecord.getValueString("orderLine.sku"); +joinedRecord.getValueString("customer.firstName"); +---- + +[source,java] +."Chain" shaped query - selecting from one parent table, a child table, and a grandchild: +---- +// TODO this needs verified for accuracy, though is a reasonable starting point as-is +// selecting from an "order" table, with a "customer" child table, and an "address" sub-table +queryInput.withTableName("order"); +queryInput.withQueryJoin(new QueryJoin("customer").withSelect(true)); +queryInput.withQueryJoin(new QueryJoin("address").withSelect(true)); +... +QRecord joinedRecord = queryOutput.getRecords().get(0); +joinedRecord.getValueString("orderNo"); +joinedRecord.getValueString("customer.firstName"); +joinedRecord.getValueString("address.street1"); +---- + +[source,java] +.QueryJoin usage example where two tables have two different joins between them: +---- +// TODO this needs verified for accuracy, though is a reasonable starting point as-is +// here there's a "fulfillmentPlan" table, which points at the order table (many-to-one, +// as an order's plan can change over time, and we keep old plans around). +// This join is named: fulfillmentPlanJoinOrder +// +// The other join is "order" pointing at its current "fulfillmentPlan" +// This join is named: orderJoinCurrentFulfillmentPlan + +// to select an order along with its current fulfillment plan: +queryInput.withTableName("order"); +queryInput.withQueryJoin(new QueryJoin(instance.getJoin("orderJoinCurrentFulfillmentPlan")) + .withSelect(true)); + +// to select an order, and all fulfillment plans for an order (1 or more records): +queryInput.withTableName("order"); +queryInput.withQueryJoin(new QueryJoin(instance.getJoin("fulfillmentPlanJoinOrder")) + .withSelect(true)); +---- + +[source,java] +.QueryJoin usage example for table with two joins to the same child table, selecting from both: +---- // given an "order" table with 2 foreign keys to a customer table (billToCustomerId and shipToCustomerId) // Note, we must supply the JoinMetaData to the QueryJoin, to drive what fields to join on in each case. +// we must also define an alias for each of the QueryJoins queryInput.withTableName("order"); queryInput.withQueryJoins(List.of( new QueryJoin(instance.getJoin("orderJoinShipToCustomer") @@ -190,11 +247,18 @@ queryInput.withQueryJoins(List.of( .withSelect(true)))); ... record.getValueString("billToCustomer.firstName") - + " placed an order for " + + " paid for an order, to be sent to " + record.getValueString("shipToCustomer.firstName") ---- +[source,java] +.Implicit QueryJoin, where unambiguous and required by QQueryFilter +---- +// TODO finish and verify +queryInput.withTableName("order"); +---- + === QueryOutput * `records` - *List of QRecord* - List of 0 or more records that match the query filter. ** _Note: If a *recordPipe* was supplied to the QueryInput, then calling `queryOutput.getRecords()` will result in an `IllegalStateException` being thrown - as the records were placed into the pipe as they were fetched, and cannot all be accessed as a single list._ diff --git a/docs/actions/RenderTemplateAction.pdf b/docs/actions/RenderTemplateAction.pdf deleted file mode 100644 index 0cc2998a..00000000 --- a/docs/actions/RenderTemplateAction.pdf +++ /dev/null @@ -1,2690 +0,0 @@ -%PDF-1.4 -% -1 0 obj -<< /Title (Untitled) -/Creator (Asciidoctor PDF 2.3.3, based on Prawn 2.4.0) -/Producer (Asciidoctor PDF 2.3.3, based on Prawn 2.4.0) -/ModDate (D:20221121094610-06'00') -/CreationDate (D:20221121094610-06'00') ->> -endobj -2 0 obj -<< /Type /Catalog -/Pages 3 0 R -/Names 9 0 R -/Outlines 22 0 R -/PageLabels 28 0 R -/PageMode /UseOutlines -/OpenAction [7 0 R /FitH 841.89] -/ViewerPreferences << /DisplayDocTitle true ->> ->> -endobj -3 0 obj -<< /Type /Pages -/Count 1 -/Kids [7 0 R] ->> -endobj -4 0 obj -<< /Length 2 ->> -stream -q - -endstream -endobj -5 0 obj -<< /Type /Page -/Parent 3 0 R -/MediaBox [0 0 595.28 841.89] -/CropBox [0 0 595.28 841.89] -/BleedBox [0 0 595.28 841.89] -/TrimBox [0 0 595.28 841.89] -/ArtBox [0 0 595.28 841.89] -/Contents 4 0 R -/Resources << /ProcSet [/PDF /Text /ImageB /ImageC /ImageI] ->> ->> -endobj -6 0 obj -<< /Length 23435 ->> -stream -q -/DeviceRGB cs -0.2 0.2 0.2 scn -/DeviceRGB CS -0.2 0.2 0.2 SCN - -BT -48.24 782.394 Td -/F2.0 22 Tf -[<52656e64657254> 29.78516 <656d706c61746541> 20.01953 <6374696f6e>] TJ -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -2.26168 Tw - -BT -48.24 753.206 Td -/F1.0 10.5 Tf -<54686520> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.69412 0.12941 0.27451 scn -0.69412 0.12941 0.27451 SCN - -2.26168 Tw - -BT -71.92168 753.206 Td -/F4.0 10.5 Tf -<52656e64657254656d706c617465416374696f6e> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -2.26168 Tw - -BT -176.92168 753.206 Td -/F1.0 10.5 Tf -<20706572666f726d7320746865206a6f62206f662074616b696e6720612074656d706c617465202d20746861742069732c206120737472696e67206f6620636f64652c20696e2061> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -1.3896 Tw - -BT -48.24 737.426 Td -/F1.0 10.5 Tf -<74656d706c6174696e67206c616e67756167652c207375636820617320> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.25882 0.5451 0.79216 scn -0.25882 0.5451 0.79216 SCN - -1.3896 Tw - -BT -200.84038 737.426 Td -/F1.0 10.5 Tf -[<56> 60.05859 <656c6f63697479>] TJ -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -1.3896 Tw - -BT -240.35126 737.426 Td -/F1.0 10.5 Tf -<2c20616e64206d657267696e672069742077697468206120736574206f66206461746120286b6e6f776e206173206120636f6e74657874292c20746f> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -48.24 721.646 Td -/F1.0 10.5 Tf -<70726f6475636520736f6d65207573696e672d666163696e67206f75747075742c2073756368206173206120537472696e67206f662048544d4c2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -48.24 681.806 Td -/F2.0 18 Tf -<4578616d706c6573> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -48.24 647.066 Td -/F2.0 13 Tf -[<43616e6f6e6963616c2046> 40.03906 <6f726d>] TJ -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -q -0.96078 0.96078 0.96078 scn -52.24 632.47 m -543.04 632.47 l -545.24914 632.47 547.04 630.67914 547.04 628.47 c -547.04 496.55 l -547.04 494.34086 545.24914 492.55 543.04 492.55 c -52.24 492.55 l -50.03086 492.55 48.24 494.34086 48.24 496.55 c -48.24 628.47 l -48.24 630.67914 50.03086 632.47 52.24 632.47 c -h -f -0.8 0.8 0.8 SCN -0.75 w -52.24 632.47 m -543.04 632.47 l -545.24914 632.47 547.04 630.67914 547.04 628.47 c -547.04 496.55 l -547.04 494.34086 545.24914 492.55 543.04 492.55 c -52.24 492.55 l -50.03086 492.55 48.24 494.34086 48.24 496.55 c -48.24 628.47 l -48.24 630.67914 50.03086 632.47 52.24 632.47 c -h -S -Q -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -59.24 609.645 Td -/F3.0 11 Tf -<52656e64657254656d706c617465496e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -163.74 609.645 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -169.24 609.645 Td -/F3.0 11 Tf -<696e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -196.74 609.645 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -202.24 609.645 Td -/F3.0 11 Tf -<3d> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -207.74 609.645 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.0 0.4 0.6 scn -0.0 0.4 0.6 SCN - -BT -213.24 609.645 Td -/F3.0 11 Tf -<6e6577> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -229.74 609.645 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -235.24 609.645 Td -/F3.0 11 Tf -<52656e64657254656d706c617465496e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -339.74 609.645 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -345.24 609.645 Td -/F3.0 11 Tf -<71496e7374616e6365> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -394.74 609.645 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -400.24 609.645 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -59.24 594.905 Td -/F3.0 11 Tf -<696e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -86.74 594.905 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -92.24 594.905 Td -/F3.0 11 Tf -<73657453657373696f6e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -147.24 594.905 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -152.74 594.905 Td -/F3.0 11 Tf -<73657373696f6e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -191.24 594.905 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -196.74 594.905 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -59.24 580.165 Td -/F3.0 11 Tf -<696e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -86.74 580.165 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -92.24 580.165 Td -/F3.0 11 Tf -<736574436f6465> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -130.74 580.165 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -136.24 580.165 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -141.74 580.165 Td -/F3.0 11 Tf -<48656c6c6f2c20247b6e616d657d> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -218.74 580.165 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -224.24 580.165 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -229.74 580.165 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -59.24 565.425 Td -/F3.0 11 Tf -<696e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -86.74 565.425 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -92.24 565.425 Td -/F3.0 11 Tf -<73657454656d706c61746554797065> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -174.74 565.425 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -180.24 565.425 Td -/F3.0 11 Tf -<54656d706c61746554797065> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -246.24 565.425 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -251.74 565.425 Td -/F3.0 11 Tf -<56454c4f43495459> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -295.74 565.425 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -301.24 565.425 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -59.24 550.685 Td -/F3.0 11 Tf -<696e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -86.74 550.685 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -92.24 550.685 Td -/F3.0 11 Tf -<736574436f6e74657874> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -147.24 550.685 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.0 0.46667 0.53333 scn -0.0 0.46667 0.53333 SCN - -BT -152.74 550.685 Td -/F3.0 11 Tf -<4d6170> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -169.24 550.685 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -174.74 550.685 Td -/F3.0 11 Tf -<6f66> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -185.74 550.685 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -191.24 550.685 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -196.74 550.685 Td -/F3.0 11 Tf -<6e616d65> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -218.74 550.685 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -224.24 550.685 Td -/F3.0 11 Tf -<2c> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -229.74 550.685 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -235.24 550.685 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -240.74 550.685 Td -/F3.0 11 Tf -<446172696e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -268.24 550.685 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -273.74 550.685 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -279.24 550.685 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -284.74 550.685 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -59.24 535.945 Td -/F3.0 11 Tf -<52656e64657254656d706c6174654f7574707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -169.24 535.945 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -174.74 535.945 Td -/F3.0 11 Tf -<6f7574707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -207.74 535.945 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -213.24 535.945 Td -/F3.0 11 Tf -<3d> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -218.74 535.945 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.0 0.4 0.6 scn -0.0 0.4 0.6 SCN - -BT -224.24 535.945 Td -/F3.0 11 Tf -<6e6577> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -240.74 535.945 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -246.24 535.945 Td -/F3.0 11 Tf -<52656e64657254656d706c617465416374696f6e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -356.24 535.945 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -361.74 535.945 Td -/F3.0 11 Tf -<65786563757465> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -400.24 535.945 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -405.74 535.945 Td -/F3.0 11 Tf -<696e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -433.24 535.945 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -438.74 535.945 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.0 0.46667 0.53333 scn -0.0 0.46667 0.53333 SCN - -BT -59.24 521.205 Td -/F3.0 11 Tf -<537472696e67> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -92.24 521.205 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -97.74 521.205 Td -/F3.0 11 Tf -<726573756c74> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -130.74 521.205 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -136.24 521.205 Td -/F3.0 11 Tf -<3d> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -141.74 521.205 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -147.24 521.205 Td -/F3.0 11 Tf -<6f7574707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -180.24 521.205 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -185.74 521.205 Td -/F3.0 11 Tf -<676574526573756c74> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -235.24 521.205 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -240.74 521.205 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -246.24 521.205 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -59.24 506.465 Td -/F3.0 11 Tf -<617373657274457175616c73> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -125.24 506.465 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -130.74 506.465 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -136.24 506.465 Td -/F3.0 11 Tf -<48656c6c6f2c20446172696e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -202.24 506.465 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -207.74 506.465 Td -/F3.0 11 Tf -<2c> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -213.24 506.465 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -218.74 506.465 Td -/F3.0 11 Tf -<726573756c74> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -251.74 506.465 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -257.24 506.465 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -48.24 461.866 Td -/F2.0 13 Tf -[<436f6e76656e69656e742046> 40.03906 <6f726d>] TJ -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -q -0.96078 0.96078 0.96078 scn -52.24 447.27 m -543.04 447.27 l -545.24914 447.27 547.04 445.47914 547.04 443.27 c -547.04 385.05 l -547.04 382.84086 545.24914 381.05 543.04 381.05 c -52.24 381.05 l -50.03086 381.05 48.24 382.84086 48.24 385.05 c -48.24 443.27 l -48.24 445.47914 50.03086 447.27 52.24 447.27 c -h -f -0.8 0.8 0.8 SCN -0.75 w -52.24 447.27 m -543.04 447.27 l -545.24914 447.27 547.04 445.47914 547.04 443.27 c -547.04 385.05 l -547.04 382.84086 545.24914 381.05 543.04 381.05 c -52.24 381.05 l -50.03086 381.05 48.24 382.84086 48.24 385.05 c -48.24 443.27 l -48.24 445.47914 50.03086 447.27 52.24 447.27 c -h -S -Q -0.0 0.46667 0.53333 scn -0.0 0.46667 0.53333 SCN - -BT -59.24 424.445 Td -/F3.0 11 Tf -<537472696e67> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -92.24 424.445 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -97.74 424.445 Td -/F3.0 11 Tf -<726573756c74> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -130.74 424.445 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -136.24 424.445 Td -/F3.0 11 Tf -<3d> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -141.74 424.445 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -147.24 424.445 Td -/F3.0 11 Tf -<52656e64657254656d706c617465416374696f6e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -257.24 424.445 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -262.74 424.445 Td -/F3.0 11 Tf -<72656e64657256656c6f63697479> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -339.74 424.445 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -345.24 424.445 Td -/F3.0 11 Tf -<696e707574> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -372.74 424.445 Td -/F3.0 11 Tf -<2c> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -378.24 424.445 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.0 0.46667 0.53333 scn -0.0 0.46667 0.53333 SCN - -BT -383.74 424.445 Td -/F3.0 11 Tf -<4d6170> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -400.24 424.445 Td -/F3.0 11 Tf -<2e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -405.74 424.445 Td -/F3.0 11 Tf -<6f66> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -416.74 424.445 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -422.24 424.445 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -427.74 424.445 Td -/F3.0 11 Tf -<6e616d65> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -449.74 424.445 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -455.24 424.445 Td -/F3.0 11 Tf -<2c> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -460.74 424.445 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -466.24 424.445 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -471.74 424.445 Td -/F3.0 11 Tf -<446172696e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -499.24 424.445 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -504.74 424.445 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -510.24 424.445 Td -/F3.0 11 Tf -<2c> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -515.74 424.445 Td -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -59.24 409.705 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -64.74 409.705 Td -/F3.0 11 Tf -<48656c6c6f2c20247b6e616d657d> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -141.74 409.705 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -147.24 409.705 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -152.74 409.705 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -59.24 394.965 Td -/F3.0 11 Tf -<617373657274457175616c73> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -125.24 394.965 Td -/F3.0 11 Tf -<28> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -130.74 394.965 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -136.24 394.965 Td -/F3.0 11 Tf -<48656c6c6f2c20446172696e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.8 0.2 0.0 scn -0.8 0.2 0.0 SCN - -BT -202.24 394.965 Td -/F3.0 11 Tf -<22> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -207.74 394.965 Td -/F3.0 11 Tf -<2c> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -213.24 394.965 Td -/F3.0 11 Tf -<20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -218.74 394.965 Td -/F3.0 11 Tf -<726573756c74> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -251.74 394.965 Td -/F3.0 11 Tf -<29> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -257.24 394.965 Td -/F3.0 11 Tf -<3b> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -48.24 345.026 Td -/F2.0 18 Tf -[<52656e64657254> 29.78516 <656d706c617465496e707574>] TJ -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - --0.5 Tc - -0.0 Tc - --0.5 Tc -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -56.8805 317.006 Td -/F1.0 10.5 Tf - Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - -0.0 Tc - -1.58443 Tw - -BT -66.24 317.006 Td -ET - - -0.0 Tw -0.69412 0.12941 0.27451 scn -0.69412 0.12941 0.27451 SCN - -1.58443 Tw - -BT -66.24 317.006 Td -/F3.0 10.5 Tf -<636f6465> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -1.58443 Tw - -BT -87.24 317.006 Td -/F1.0 10.5 Tf -<202d20> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -1.58443 Tw - -BT -99.10287 317.006 Td -/F2.0 10.5 Tf -<537472696e672c205265717569726564> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -1.58443 Tw - -BT -188.0788 317.006 Td -/F1.0 10.5 Tf -<202d20537472696e67206f662074656d706c61746520636f646520746f2062652072656e64657265642c20696e207468652074656d706c6174696e67206c616e6775616765> Tj -ET - - -0.0 Tw -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - -BT -66.24 301.226 Td -ET - -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -66.24 301.226 Td -/F1.0 10.5 Tf -[<7370656369666965642062> 20.01953 <792074686520>] TJ -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.69412 0.12941 0.27451 scn -0.69412 0.12941 0.27451 SCN - -BT -147.10029 301.226 Td -/F3.0 10.5 Tf -<74797065> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -168.10029 301.226 Td -/F1.0 10.5 Tf -[<20706172> 20.01953 <616d657465722e>] TJ -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - --0.5 Tc - -0.0 Tc - --0.5 Tc -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -56.8805 279.446 Td -/F1.0 10.5 Tf - Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - -0.0 Tc - -BT -66.24 279.446 Td -ET - -0.69412 0.12941 0.27451 scn -0.69412 0.12941 0.27451 SCN - -BT -66.24 279.446 Td -/F3.0 10.5 Tf -<74797065> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -87.24 279.446 Td -/F1.0 10.5 Tf -<202d20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -95.934 279.446 Td -/F2.0 10.5 Tf -[<456e756d206f662056454c4f43495459> 80.07812 <2c205265717569726564>] TJ -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -251.97368 279.446 Td -/F1.0 10.5 Tf -<202d2053706563696669657320746865206c616e6775616765206f66207468652074656d706c61746520636f64652e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - --0.5 Tc - -0.0 Tc - --0.5 Tc -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -56.8805 257.666 Td -/F1.0 10.5 Tf - Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - -0.0 Tc - -BT -66.24 257.666 Td -ET - -0.69412 0.12941 0.27451 scn -0.69412 0.12941 0.27451 SCN - -BT -66.24 257.666 Td -/F3.0 10.5 Tf -<636f6e74657874> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -102.99 257.666 Td -/F1.0 10.5 Tf -<202d20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -111.684 257.666 Td -/F2.0 10.5 Tf -<4d6170206f6620537472696e6720> Tj -/F2.1 10.5 Tf -<2120> Tj -/F2.0 10.5 Tf -<4f626a656374> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -233.694 257.666 Td -/F1.0 10.5 Tf -<202d204461746120746f206265206d61646520617661696c61626c6520746f207468652074656d706c61746520647572696e672072656e646572696e672e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -48.24 217.826 Td -/F2.0 18 Tf -[<52656e64657254> 29.78516 <656d706c6174654f7574707574>] TJ -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - --0.5 Tc - -0.0 Tc - --0.5 Tc -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -56.8805 189.806 Td -/F1.0 10.5 Tf - Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn - -0.0 Tc - -BT -66.24 189.806 Td -ET - -0.69412 0.12941 0.27451 scn -0.69412 0.12941 0.27451 SCN - -BT -66.24 189.806 Td -/F3.0 10.5 Tf -<726573756c74> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -97.74 189.806 Td -/F1.0 10.5 Tf -<202d20> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -106.434 189.806 Td -/F2.0 10.5 Tf -<537472696e67> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -138.8685 189.806 Td -/F1.0 10.5 Tf -<202d20526573756c74206f662072656e646572696e672074686520696e7075742074656d706c61746520616e6420636f6e746578742e> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -q -0.0 0.0 0.0 scn -0.0 0.0 0.0 SCN -1 w -0 J -0 j -[] 0 d -/Stamp1 Do -0.2 0.2 0.2 scn -0.2 0.2 0.2 SCN - -BT -541.009 14.263 Td -/F1.0 9 Tf -<31> Tj -ET - -0.0 0.0 0.0 SCN -0.0 0.0 0.0 scn -Q -Q - -endstream -endobj -7 0 obj -<< /Type /Page -/Parent 3 0 R -/MediaBox [0 0 595.28 841.89] -/CropBox [0 0 595.28 841.89] -/BleedBox [0 0 595.28 841.89] -/TrimBox [0 0 595.28 841.89] -/ArtBox [0 0 595.28 841.89] -/Contents 6 0 R -/Resources << /ProcSet [/PDF /Text /ImageB /ImageC /ImageI] -/Font << /F2.0 11 0 R -/F1.0 12 0 R -/F4.0 13 0 R -/F3.0 17 0 R -/F2.1 20 0 R ->> -/XObject << /Stamp1 30 0 R ->> ->> -/Annots [14 0 R] ->> -endobj -8 0 obj -[7 0 R /XYZ 0 841.89 null] -endobj -9 0 obj -<< /Type /Names -/Dests 10 0 R ->> -endobj -10 0 obj -<< /Names [(__anchor-top) 29 0 R (_canonical_form) 16 0 R (_convenient_form) 18 0 R (_examples) 15 0 R (_rendertemplateaction) 8 0 R (_rendertemplateinput) 19 0 R (_rendertemplateoutput) 21 0 R] ->> -endobj -11 0 obj -<< /Type /Font -/BaseFont /dc156d+NotoSerif-Bold -/Subtype /TrueType -/FontDescriptor 33 0 R -/FirstChar 32 -/LastChar 255 -/Widths 35 0 R -/ToUnicode 34 0 R ->> -endobj -12 0 obj -<< /Type /Font -/BaseFont /90d1b9+NotoSerif -/Subtype /TrueType -/FontDescriptor 37 0 R -/FirstChar 32 -/LastChar 255 -/Widths 39 0 R -/ToUnicode 38 0 R ->> -endobj -13 0 obj -<< /Type /Font -/BaseFont /f7f4ef+mplus1mn-bold -/Subtype /TrueType -/FontDescriptor 41 0 R -/FirstChar 32 -/LastChar 255 -/Widths 43 0 R -/ToUnicode 42 0 R ->> -endobj -14 0 obj -<< /Border [0 0 0] -/A << /Type /Action -/S /URI -/URI (https://velocity.apache.org/engine/1.7/user-guide.html) ->> -/Subtype /Link -/Rect [200.84038 734.36 240.35126 748.64] -/Type /Annot ->> -endobj -15 0 obj -[7 0 R /XYZ 0 705.83 null] -endobj -16 0 obj -[7 0 R /XYZ 0 665.75 null] -endobj -17 0 obj -<< /Type /Font -/BaseFont /ae1e8f+mplus1mn-regular -/Subtype /TrueType -/FontDescriptor 45 0 R -/FirstChar 32 -/LastChar 255 -/Widths 47 0 R -/ToUnicode 46 0 R ->> -endobj -18 0 obj -[7 0 R /XYZ 0 480.55 null] -endobj -19 0 obj -[7 0 R /XYZ 0 369.05 null] -endobj -20 0 obj -<< /Type /Font -/BaseFont /adefa7+NotoSerif-Bold -/Subtype /TrueType -/FontDescriptor 49 0 R -/FirstChar 32 -/LastChar 255 -/Widths 51 0 R -/ToUnicode 50 0 R ->> -endobj -21 0 obj -[7 0 R /XYZ 0 241.85 null] -endobj -22 0 obj -<< /Type /Outlines -/Count 5 -/First 23 0 R -/Last 24 0 R ->> -endobj -23 0 obj -<< /Title -/Parent 22 0 R -/Count 0 -/Next 24 0 R -/Dest [7 0 R /XYZ 0 841.89 null] ->> -endobj -24 0 obj -<< /Title -/Parent 22 0 R -/Count 3 -/First 25 0 R -/Last 27 0 R -/Prev 23 0 R -/Dest [7 0 R /XYZ 0 841.89 null] ->> -endobj -25 0 obj -<< /Title -/Parent 24 0 R -/Count 0 -/Next 26 0 R -/Dest [7 0 R /XYZ 0 705.83 null] ->> -endobj -26 0 obj -<< /Title -/Parent 24 0 R -/Count 0 -/Next 27 0 R -/Prev 25 0 R -/Dest [7 0 R /XYZ 0 369.05 null] ->> -endobj -27 0 obj -<< /Title -/Parent 24 0 R -/Count 0 -/Prev 26 0 R -/Dest [7 0 R /XYZ 0 241.85 null] ->> -endobj -28 0 obj -<< /Nums [0 << /P (1) ->>] ->> -endobj -29 0 obj -[7 0 R /XYZ 0 841.89 null] -endobj -30 0 obj -<< /Type /XObject -/Subtype /Form -/BBox [0 0 595.28 841.89] -/Length 165 ->> -stream -q -/DeviceRGB cs -0.0 0.0 0.0 scn -/DeviceRGB CS -0.0 0.0 0.0 SCN -1 w -0 J -0 j -[] 0 d -q -/DeviceRGB CS -0.86667 0.86667 0.86667 SCN -0.25 w -48.24 30.0 m -547.04 30.0 l -S -Q -Q - -endstream -endobj -31 0 obj -<< /Type /XObject -/Subtype /Form -/BBox [0 0 595.28 841.89] -/Length 165 ->> -stream -q -/DeviceRGB cs -0.0 0.0 0.0 scn -/DeviceRGB CS -0.0 0.0 0.0 SCN -1 w -0 J -0 j -[] 0 d -q -/DeviceRGB CS -0.86667 0.86667 0.86667 SCN -0.25 w -48.24 30.0 m -547.04 30.0 l -S -Q -Q - -endstream -endobj -32 0 obj -<< /Length1 11940 -/Length 7523 -/Filter [/FlateDecode] ->> -stream -xz |[ՙ9^=,?$zؖ-]ZeYmْߖmɱ/!'!bɏDK@^8 44)e gm.0!ev; ہI~L+ٱMv{wyF!"{껚O 'f~x{"/olo߰ah ^D'W9j"A1h57^7BI@t{$0$2tBt2|P;( -G­g ʛ";f3wADB?=h23X7_L_ `܉X[x[_!nepv:}o/ -AV`oY - 4tGG3J -.E@ďlWhN "SHP< uц 1G먍'z FH-h<ëGWEFT/x7Y'P3KygxzmU!r ?|S ?َ ~DWCt+-2򆑑HɳOP3ٌLPWKADx&\SY@[k[[ B7g@ ͸韽6W$LN@:FJ+!DS..| +32+da>IYBڑ/*au؞i<[j$B$9:: Qf}/#үjh_P uz/8PAOI8GA( I 4i8t ceҠS:J^)Sj =BJr%;`ڛH 8M&ڇhY`$;J˱5Jylut-kEѥԌjRj:1Z"u0k5{UDŽQfN8|m{'5U7! T/Xg&:MMl+Ůu՞jz^t{YLZ!xvf/$V@g,EC <Np^7F7&'(<y1rK,dx")^}L3d:^9}4?<}iy_>Ӹ䎓D'Zrx^\ -jI4L\hw8n "P |;˷ -,ȫ$)QdFAA^M9$ɆrxHbkopГDtQ 嘇/ ^` jA~(ogڲշã;K09KFAv`@HJ}4M܅2C)& -9xsscx;+ceK+3q;>gq -K iBxyoݵRQ:{n[NJSKGo{[؅7KI-aO6⾕ ,voĈǶݪRG\%~C\$^%/Eb#-7K`8?\4yTO)KJwAYpx8ƞ5l5ԧD-a #^[ڳ[ޗ7Nqp79؅gwD51O"w?T' HX S yFgCoa{ 0 I8yxiط ⵍǜih[o8k9P5v[Q;4wrA#lnsҫ%ufU~\QZktwM>>5";`TE4bVB8ȕbmފI i!Q+) /sl`p!Xc(-nӅ,~LB&(P*'oRU(4Xo4kr"|В~[S<Y,(!mKm*XC&sE|K}_&|[p{_7h-agPisǙNmVuiG_Mu,y,}}ZַU*m:IzW/`Vl#ɅMV,4D3>yXMᏓEw+aaUaEMqF8`//ydع`Yu8bDYeT%z0Qc]W%l7T.S}7/5Bek+)[ ,mG2k*R7,e}nMZ2Cq./OP8x=ulұ_n 2Z_J*Z;JM=xwpʾ.>'o0a|s|aG -3:)ypX*yo]H.} gK8`@Рz2WsXݻ6y'$|'?Nu/D9ָ$;Yv^e2}Inn^6LZ, DرD7fmQ_XwEW'/k{,՘1v&eU{OwOحNųySj#{7H5-m cb+T$"J/E6|Dsvm=ûOn1l:t)5oq.orSϣoIkƫk,RfmSZZq[[ 7Ci46TWPb񔛶Lv0RcV6Ybsid+c\?ci1S9 a1oh6"j.!QfEE_+_sk+Җ-2W#yv -ʎM9 B2Ve# 0_5Ul)5V/j#ob=R+Lg6\s#gE6\\|R0H|K_]W <_c2C{Jcn^^ 5x٥F#X7JւY8=2lvp5uűxs<>?BYTѨHʩ8~{ʞ*ʁЖ-:*˞TuVUɧaٺqc<+tnWo^SSyڀ<^։m{ˡkZED_f0sg墳mS}^a%pFps?]~Pd&SLkijZG %/U9bhkPZ[uZ9D&/)1 xg uLn.Sgו][9mR -#7MP`HH91zZZ6=ƜFUI?"Gڒ޴)ۥN[ZuRj=kFڇi3@ڕ<z+Y&O}V5խ-Ve̓Zi0 v-?=#{=ޭjkdgK$'nՌZL5%@Ug!,=hTԿ}['z|?aݘpETT!#ؿv~t s' ҰS|cg=XzسPmr;P-'ɏwO)wNDϜLw.;^0Wj;`ߖKS+T/ sfLg+2L4O};8h,$͋hy]WpוZ'+:{;G;Ff0},8yʛr>unޞ̀ێ:W- ;_[k:{-{Mf%/k辭a.cwM?P9Z.qEDzKJ#)?2dU L軧NÑBk{yjpEx K) yHI -!#嘺Pi/LC"sOEʎf+Ca())UdSr}="R7Tf*#mQ o,wK<'̎syp _wEύ JvsUAaC{`z,pY`f먦٬,l b^Kyignf:7[j.,iv_w m55ʒPСmUx.bbwEJvMƭr2-qx3=g'խ0?yL;PʙFgfс=,2Z|'y2Ybij]6pNGDf@nq'/^27yW5j~X[P@Nv>X%#ԯ7s-xL-__婬j/ګW.3%¹ nWyj5&IӰUI/`E"yl@I>+D2KledƪH#6PXSe"lU;F&[Mggx4jOuDgNMLʪjzC4:ў1 lUB{CAKO4٩h89:H`bjz67hVb]q84NUJ{p4yE~*Fl fwհ+C+ГSq懂}p0%ad<>`~ 3X4bŭ s\p*:8 ;q 6B1sn:!D΄S›RMTY[+ NS;$ӴK{ZV#ޡAzm`gҽtGo3R'3Jw{z\th -ҡ;ffC"3)UP蛱є%LO&ld*x Ε'{ClEf9a8ff1Kl*lNX{;7E3hEShM8aT +q5P6 -0 -A} - cn q|Ck/ а{q}* -mh3|} ?o -P6s+ vŀDM̾;O$&aE7K8?cQ B)m]sX&9ʳ<Nu5i.Bqi݊nn HݴX krdiQn:b,9kX/wrq.Vg㠅th%aAy"9\Yk&9Y/xv|{^h޴FCpnUW#Ȃ7eN@OK po- -/w/lqXQ/[n2({k d/p]\ G:׃F 5 ;RޙJwgfLqg9;"e=U Y~G51qmƸJ p`Xp#/#K/ 7<˵^]7aKcJ!p/kԳ߄V%wiq{%1*EzTg -h[ h-9M5AܠkmAwꇨ{!C#ݚ, ۶l -endstream -endobj -33 0 obj -<< /Type /FontDescriptor -/FontName /dc156d+NotoSerif-Bold -/FontFile2 32 0 R -/FontBBox [-212 -250 1306 1058] -/Flags 6 -/StemV 0 -/ItalicAngle 0 -/Ascent 1068 -/Descent -292 -/CapHeight 1462 -/XHeight 1098 ->> -endobj -34 0 obj -<< /Length 1278 -/Filter [/FlateDecode] ->> -stream -xenFὮBtHs&@nu{stԒ + }ik/y_!}t~]snVyg쾝+|<.÷}v)[eO/_q9_ɯ}ഏOmt_LLv͵LJ1w9)e6=n?[i(JKiU:JSz2QeL*EJ̔Ye,*+eU(NU-уk5x5F^ky ^#k5x5F^ky ^#kxZV^kZy-^+kxZV^kZy-^+uxu:N^:y^'uxu:N^:y^'xz^^zy=^/xz^^zy=^/o xo7 Aހ7y o xo7 Aހ7y oxo7jo$JI$$*I$ITH$$QI"IDD%$JI$$*I$ITD$$III$$u>M&Iބ7ɛ&y$o›Mx o7M&Iބ7ɛ&yUoa[5joSor4ۂsޮ>,x/x;Â31x; -JJ특xxx+!ÊwBxbx+ށr;2kΜJYeY7+|x oS7+[ƛețךyޢoV浖 -㭌"RW*4XqC^J[(^1»y]k}YM-x e#e_y+h܊7k:/Z5dUluz5n[eB&Si|y(%q& %+S%*ABijPU6\h,(+L,4G5sh>:OVRP*#i|e0U,*oU/*[U,Nmlx:&\}M)L2\%CHF7էʻBL)jƻJ1:%H߯:=S۳zrmkO媳7 -endstream -endobj -35 0 obj -[259 600 600 600 600 600 600 600 600 600 600 600 293 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 752 600 667 600 652 621 600 600 400 600 600 653 952 600 787 600 600 707 585 652 600 698 600 600 692 600 600 600 600 600 600 600 599 648 526 648 570 407 560 600 352 345 600 352 985 666 612 645 647 522 487 404 666 605 600 645 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600] -endobj -36 0 obj -<< /Length1 12240 -/Length 7636 -/Filter [/FlateDecode] ->> -stream -xz t[ՙ=,d[~D#oIeَ-%ˎI$[#-ǒCICmNd1,Ph$po SfqS>hitz44\ -]!￷$3YHK6t3`,;tҀk|=K_% 'xU~x#IJ'y$2 &gO || zv0/} u7஘F;9PΙ {)Q˥x(hЂeLY v`ީ% -.ʚfL_.b*/_L"lܞ`ofUi``U & bK^1WQGGA mX`RQ@ [,`ٿjs$tr_AwOol#N%"d wiE[\K+G^.,][QZ]^y*pPՑζwS;r JM~S5G;Z-4>g̣1b,2.RF&NjDkVfs)= ||<2B -KIV0Ugr-j}dIN >Uܤd~j`sE{2@/C? Z@Xܑ.!|P!4ƓM8iH9 aUh˼m6G G(v -}+ȴr^}*GV8r0׶L5.FTiaP1,8$)4-EdBg߈*:\Aq:&5SVꤼQ (T ϠD'[\MPT` x# JE$ۢĤzTam3PG C ) @[<P{*°0-N73h|"qPZ,pBu@l)RwnWmۣTX"ʅBۃ*IEg -1BrfEd I~Ih, Z* !b+u&}1L|M|E `yEΗ4/@ - -2K)W] trA1'FH,X$Uo!o9i_Њu\Ǣ:_^r־!.>?7hGn3gcZ><{;x|,zLcGbѣ33L3uv$j 8Π !!Ϻ~^ؿ_E(L!A>Ïww0)x%ACX| g'EED0p((NeQ蝍T))Vif3 Kя4ѽ^Fg|uϡp a.gu&WuACл*: gP!S Ij-i7>vuj[0eN4 7P&?@Cn4j#F+A+7yt9GG#Ի*+ԏBwXvCY"(X!NHs㓈%VȂg̷؊m5cf -+{VD}F.tz]Bx~bsl&`qAoj}IiOoI#y$ ٨,( |p"eldɉϔMil[[c0.n1j&=0z\ˡo>C|eGUz# LY$*,1׳`J=[55]4R]?e__٪KQ]uI)Ѐ\^a*S3dNQm[-S*?ZP*2 45zCu pN1kSML*4QPey=C=!VF( -F((ޜWAq xl*gI =n$pi N}Nkԩ՚* u!Tp%URiάUQIF yt#$*D)h#6Eج-SdLr(&FJΠT겕ȲKB'URpg?y⦯2HXv{Q{HO6OTX˲H'Tl ` *INP1k50S'J~fv23Ɓ"YMa1} -Џ 9 [QWܤ}"!1_*d֬KcBENW-)um/>i #Fe*ZnV嫟DeV?Sg|5˝ 3-||TF,<,#ϑT(8_X8uQ'<W"R)./I5wKbwɸ2x(r ?Z=ğN<zIQ:*D%M^q"-=EaNN!`jVK03jUfUX_#)NgTl5?婋uR}fm))'}yr?ttw1'F#N/ ہHt"KעT@;2E3Xg;ئKM{A}{_׎wԥڶ뢽?<߉ \;;v5sB(sZ[_Rj0-ZTgg$⃕w55G ̝'H6zB KG$yƵ&->'tYNf-9@F^>2TӻۢR5̬s3yfeISTAs>td2g5֚gvߺuٿX1p\wyp~Eg/c{et(+QScnk^FdRpf}ΠLfŒpi'4:^+ɜZQ6G1PDEJxz9s SLU*ajTdU5ΉuԔ+,-6n][]ibb~YMFN}6v{`mqQ_CCeAϸӤe%e-:0cbY:7}:Id|h♓#uan0e~p<6%#^MK*C]뚶W넄.uuy=Iqxn$*6oIkCGK2 -an{:쨷U&Fg::5PxfGT8t3KV:n"`@Ű+݈0lګvDh1 o^;n(Ԥh -3ta,2gkȒSS;>^]_`?DI {juL̙ambFCA|7YSQDlw2Wg nwfp|`g1xm JR︅ ׳$gaU6qְ(G #mRVjo>˜w8b zqJx.t?*,1 ]U6Qpx VnEc 2G*1+__{@][a逹nY;ֵKP]N7d6{Iȶ$M[pI=C$wM5uUR9LCw~k{:&Z>Շ1*F5&8'ӝGŸS%[{K$-;|证݀->["a9MR#Z7]D?bGkV[YR ]料\ɖ[ qXea< Br )K`('d;q H6+JuN oB[XK"EI8Jәqܡ0Pv7t[{7vb; pxipOq~=7螄^{f#9v:owakoq_M׺|az>7g IcOKjF>44 w -P e@B3 v4a򰟙W[i o/ OHvC<i4A&"wgQ#lej&Av<}'=w9(hK:i w$݀K-o%3W -2Eu"^(Ox<]4{6hD4hL S M?Z@e/#}@ݹ&k[,7-T>@#D<-8{s4oH?(LQl{չ 6h(ΝrzAͫ1@ tg6yDˤDp 5d7)7BóFQ\+mNp] khޝPCdn/ -{ggh&)ڏYGɯ<ȷ!&~f:"> -endobj -38 0 obj -<< /Length 1278 -/Filter [/FlateDecode] ->> -stream -xenFὮBtHs&@nu{stԒ + }ik/y_!}t~]snVyg쾝+|<.÷}v)[eO/_q9_ɯ}ഏOmt_LLv͵LJ1w9)e6=n?[i(JKiU:JSz2QeL*EJ̔Ye,*+eU(NU-уk5x5F^ky ^#k5x5F^ky ^#kxZV^kZy-^+kxZV^kZy-^+uxu:N^:y^'uxu:N^:y^'xz^^zy=^/xz^^zy=^/o xo7 Aހ7y o xo7 Aހ7y oxo7jo$JI$$*I$ITH$$QI"IDD%$JI$$*I$ITD$$III$$u>M&Iބ7ɛ&y$o›Mx o7M&Iބ7ɛ&yUoa[5joSor4ۂsޮ>,x/x;Â31x; -JJ특xxx+!ÊwBxbx+ށr;2kΜJYeY7+|x oS7+[ƛețךyޢoV浖 -㭌"RW*4XqC^J[(^1»y]k}YM-x e#e_y+h܊7k:/Z5dUluz5n[eB&Si|y(%q& %+S%*ABijPU6\h,(+L,4G5sh>:OVRP*#i|e0U,*oU/*[U,Nmlx:&\}M)L2\%CHF7էʻBL)jƻJ1:%H߯:=S۳zrmkO媳7 -endstream -endobj -39 0 obj -[259 500 500 500 500 500 500 500 346 346 500 500 250 310 250 500 500 559 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 727 500 500 500 792 500 500 500 623 937 500 500 500 500 655 543 612 500 674 500 500 500 500 500 500 500 500 500 500 562 613 492 613 535 369 538 634 319 299 584 310 944 645 577 613 500 471 451 352 634 579 861 578 564 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 361 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500 500] -endobj -40 0 obj -<< /Length1 4064 -/Length 2678 -/Filter [/FlateDecode] ->> -stream -xWkl[w;MqB}$_7v$Mi>Nbk_?B[YWF>T0j@t[6|!6mCS' AIi)m:{5P.G@(G};5]B!ji*l,r)&B_[PP6rȗJ"T#旂O^FzXxa}+E b7!/@~WZQV1}w_XCӘ/*/g'0^ѫx*MdSH Q[JTtyys3}@yg(wl{q,T6O, w_= UO9ҭKpp$$8!!_ A2L>v" 8Bu^# -4/]ӒyгpTa} m1iphű yDפ ^.vA|"(.)Ź `]yEeT?^ -ݭyEJѳeǼ=Z -1+JѬNNѫ7+H=UcBsՋtγʫJѣ .zE"S~Rl` 0:=5=SaJLȱ&*5ըiլ1P`ӰH<9㣌K{u9s .J#b^TU)1N{(%樇 ]%i ͈<Zb}L8dƠOz^G`if&.RnN' f39\dVTlj0o5+5a蛛Zk"2LB<}ehL/8WT~gvwzCدÉysW6 7biM -J/x3>{>`R:@Oj(*h'ǽصiWVU*\Y $p $FHz|^ۄzu]-Igo%] ?E(GZ{" ĊNn~4//->=1B#RmF"XEuVWduuv4mͭ- ԑx=zafA]-? DfE-0DS%ē1ၩ)̡{yI-liŢk E ZM}=yϡ{6l}ah{@;36y@eS$^u,?m.v6AiyݶTT8F=[P/I3;p$Ұ^yu`[jFIaS܁6slؗ${ 3]]i?ȭ~y^$&TՔl[?Stޡ\=(d?H̦;u BÏ+;sx3XO"Vvwwڵ3$:Prұc̖3kɮe c'N^Nl4tt*dA|MnΚm=Խ[VitUyÆW(9Ӡ#gtZrqsbG]]-1Y\we'Y}دwѯyTņfUP9^8^9y,~|8iȾ0ɐLmҜF@d!n03LT$ v;1_HZDe HY@2p#)$IYx0}ė-t"6Fr&d4A,Ř4 )DXbȡL#i=IiMd|V)An]MZB4φcVG8Fu&疬qGt>#+mebG$<֐cYnaNkhIp ۹ 8O!GnoG',KI)-ő\vǟ/Kse(y[:/=+BW~tRzDIPatQ_"`$4ܼЊ>_ -J'$1\NNZcO=X`aiE /Rd<,G?fB:(Ҍ4`ӥ&sx ;{1}/.XE -endstream -endobj -41 0 obj -<< /Type /FontDescriptor -/FontName /f7f4ef+mplus1mn-bold -/FontFile2 40 0 R -/FontBBox [0 -275 1000 1042] -/Flags 4 -/StemV 0 -/ItalicAngle 0 -/Ascent 860 -/Descent -140 -/CapHeight 860 -/XHeight 0 ->> -endobj -42 0 obj -<< /Length 1278 -/Filter [/FlateDecode] ->> -stream -xenFὮBtHs&@nu{stԒ + }ik/y_!}t~]snVyg쾝+|<.÷}v)[eO/_q9_ɯ}ഏOmt_LLv͵LJ1w9)e6=n?[i(JKiU:JSz2QeL*EJ̔Ye,*+eU(NU-уk5x5F^ky ^#k5x5F^ky ^#kxZV^kZy-^+kxZV^kZy-^+uxu:N^:y^'uxu:N^:y^'xz^^zy=^/xz^^zy=^/o xo7 Aހ7y o xo7 Aހ7y oxo7jo$JI$$*I$ITH$$QI"IDD%$JI$$*I$ITD$$III$$u>M&Iބ7ɛ&y$o›Mx o7M&Iބ7ɛ&yUoa[5joSor4ۂsޮ>,x/x;Â31x; -JJ특xxx+!ÊwBxbx+ށr;2kΜJYeY7+|x oS7+[ƛețךyޢoV浖 -㭌"RW*4XqC^J[(^1»y]k}YM-x e#e_y+h܊7k:/Z5dUluz5n[eB&Si|y(%q& %+S%*ABijPU6\h,(+L,4G5sh>:OVRP*#i|e0U,*oU/*[U,Nmlx:&\}M)L2\%CHF7էʻBL)jƻJ1:%H߯:=S۳zrmkO媳7 -endstream -endobj -43 0 obj -[500 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 500 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 500 364 500 364 364 364 364 364 364 364 364 364 364 364 364 500 364 500 500 500 364 364 364 500 364 364 500 500 500 500 500 364 500 364 500 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364] -endobj -44 0 obj -<< /Length1 6184 -/Length 4155 -/Filter [/FlateDecode] ->> -stream -xX}P[וI0$x`xF I| _d!,K $8:דN;Y7Ӭxdwuݶi37z]7d24tv$ms{L<{{s9'B}F?!ߤwꅕ_=3cY]Q)LCHQ cS㓙*.#$;LE( ؊5hfP!{L -t+b -?brrS:(?ۀEI -=a~%T'UT&-)a5d]qqhlx1&=HC6 ~h!h硽 Zh,4+&hZ4ԬX$Y5j5$"cOCZ~izUtN:[C!|@˧RP*W̙)U&U-ȕVkע -9M_\--C֗#HG]ws'UqEC{w8۰J:^<\gt]Nw+2]hot5F>.Q M/=y^/O']:^n>нV8dktP.KmSu~ȿ t\\#;y9BTi49Yp2O4w*nk휛o!J2,ŸǸO x8>2o dЕh;1pu6nʖ;<ı::b"䡏+6d*b!f@$l_P+MFZivs%g֊+SUц L9 ǒWF9&c=&%cJUiZeDڃӪNxHZ?b/N {`BmR*ZҨ|g7p's᷍l=ī2]뮃xx"qx/>;ggm̀Ͷ -UMؠ< &gOL; ;d;2s.Q0* *|L9JEVe(E -~} G/'^1<|Xo;]C B̗p9DZ D6FCuچ,Ipu9>H;ϔ3 bM`-6i~!t .؜\f_R1c8ڠ~915֍ln3/ՙouwl-ph-0F>@Pacͦ%{+LlA ۱k9d8@fy ލ}Kzɂ|1kp5@U'^H|md(M'lTWUQl鹶i3TSWiɤ싀,(PJOlkwOsu-UسK6> 7>rW'w` *Ro[%;n*m#cU8y'u0mA:o_Z%uTUV6D<)om?a~O6Zm67h3Nw:IlFЍ֌li) tU]s=]K;wΉߨvh^ 쨅WA/ʃh6ZZ*V!cM݊z'p_Y]R\bkqܝɢ#{D3?.ăC72_oBmŭ Y?77'W l…c3//,FE0DYG>װ=Svӄ/S%WSR~EDDlNXY?.&.`ME7# zݨpXY{"]c R' Zq]Hki! -k4P~&a[$ dD`BYP" - HtZIJtEZ8(*À]J4FMfPWϿI3^P +!f$ZK*!U?+%V ͑=ӡ`̷3 -rf+'w Gg4FNhW鉼82 E\A@ez|rjbv&"X0XQAcJـdIH3^ltYBSo"pvAb)F8/Z VZ`^"xnO]2N/% a=?υP _-xRؾ*#?^~($ l p2*Swj/$A ~{hr׳v2 >aLxZsDqx|l pOAaw(9(&V 3VfoЋ @χ$8-(jg@#-*%mV@J -S n6Eh8:*J_KTK -endstream -endobj -45 0 obj -<< /Type /FontDescriptor -/FontName /ae1e8f+mplus1mn-regular -/FontFile2 44 0 R -/FontBBox [0 -270 1000 1025] -/Flags 4 -/StemV 0 -/ItalicAngle 0 -/Ascent 860 -/Descent -140 -/CapHeight 860 -/XHeight 0 ->> -endobj -46 0 obj -<< /Length 1278 -/Filter [/FlateDecode] ->> -stream -xenFὮBtHs&@nu{stԒ + }ik/y_!}t~]snVyg쾝+|<.÷}v)[eO/_q9_ɯ}ഏOmt_LLv͵LJ1w9)e6=n?[i(JKiU:JSz2QeL*EJ̔Ye,*+eU(NU-уk5x5F^ky ^#k5x5F^ky ^#kxZV^kZy-^+kxZV^kZy-^+uxu:N^:y^'uxu:N^:y^'xz^^zy=^/xz^^zy=^/o xo7 Aހ7y o xo7 Aހ7y oxo7jo$JI$$*I$ITH$$QI"IDD%$JI$$*I$ITD$$III$$u>M&Iބ7ɛ&y$o›Mx o7M&Iބ7ɛ&yUoa[5joSor4ۂsޮ>,x/x;Â31x; -JJ특xxx+!ÊwBxbx+ށr;2kΜJYeY7+|x oS7+[ƛețךyޢoV浖 -㭌"RW*4XqC^J[(^1»y]k}YM-x e#e_y+h܊7k:/Z5dUluz5n[eB&Si|y(%q& %+S%*ABijPU6\h,(+L,4G5sh>:OVRP*#i|e0U,*oU/*[U,Nmlx:&\}M)L2\%CHF7էʻBL)jƻJ1:%H߯:=S۳zrmkO媳7 -endstream -endobj -47 0 obj -[500 364 500 364 500 364 364 364 500 500 364 364 500 364 500 364 364 364 364 364 364 364 364 364 364 364 364 500 364 500 364 364 364 500 364 500 500 500 364 364 500 500 364 364 500 500 364 500 364 364 500 500 500 364 500 364 364 500 364 364 364 364 364 364 364 500 364 500 500 500 500 500 364 500 364 364 500 500 500 500 500 500 500 500 500 500 364 500 500 500 364 500 364 500 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364 364] -endobj -48 0 obj -<< /Length1 6560 -/Length 3565 -/Filter [/FlateDecode] ->> -stream -xW}pSWv?=}S2 x@!ٖd Id@GXL 1$MgHۙR ΤfaN?8ӖLƉQϽz26km={{9>jaD w,G$hd<%g[;lϫ47&@ p4#q93[h_s E -OJ PДA߻ߕHZG~"3.CϐײJ.1i9uK%m@šl&>y珠3Sr;8lOW9k#}K6Ҵ;LgkJY{,9"Ÿ ~Fm3hZDJ@`x$ %`|؉Qb?,hCi'WPPz WPYf8b& ,B#I|E} ' -0Ѐшt #ZJaq/&qgz5fw3ä^&͈lB& Å[4ؖAHV$W֕h]"_!݇pm|mP9XQV ARc8V)+p -_*q3ЄM@{bF1(r>9r8W&XYg6y5߭86bsܒyH\$N1YY-`eBUd;:}G dHObu;r{noߟmNl"ݺ{csMs';Hygz#scX%.gE,Y%t ]?w]8~ww/^e‡KsMfCɄ܉/}1Wl8&brG GF?7wn -q\UDz9=5sJ@600n:5h+H\M|z*.Zb&,wL1rm.omX0//ݸ  -W7,NrKç n'½wc6~[]_JE"svO}u;!L8|"aWy`:+ޅdZvw6lb2K;^0N9?}t#_I—7nk[2uB+@%r%Ziv·uV8F&M :mF:mA|N+>J*|[VrZkt-losCrg4zѢ#ni.6bB(NQ:mI+`Z+Acr e[n+u7% طd -;he<UJ̨WrXOf"R&Qt' -i<ŞRrd&Mw;$G|ipUdd*wh&6ߘsVx9U&iq%JǦrrGBUG .td㙔3NT8׈S9WGk4 $0do&R&^s -[Dr\IqtTQ5P$*ɒB}tcgd^%&q554>Ni_(GG>ޡA/DKt7nJQlNi&GDʯ(ώ2x2r:>)qY%JB~(?Tey%fL$z²rdVw\9= ,LA@)cO'@A·}-nD&p]sNQA_籏 Z(9 ?җpLsFP%#ߎm#t,=űT=Fa } > -endobj -50 0 obj -<< /Length 226 -/Filter [/FlateDecode] ->> -stream -x]j >E B^rШswt-0y/NX -ni/a%D6Hp֛ݮ&3Ntl1J1AV'f|`8,!.pL=\1VLkpiыɯfE;P31GF]069ܲXL\)!^3%)LI:=M(;u~xeR(R?CRKnGo -endstream -endobj -51 0 obj -[259 1000 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600] -endobj -xref -0 52 -0000000000 65535 f -0000000015 00000 n -0000000240 00000 n -0000000441 00000 n -0000000498 00000 n -0000000549 00000 n -0000000821 00000 n -0000024309 00000 n -0000024705 00000 n -0000024747 00000 n -0000024795 00000 n -0000025009 00000 n -0000025179 00000 n -0000025344 00000 n -0000025513 00000 n -0000025714 00000 n -0000025757 00000 n -0000025800 00000 n -0000025972 00000 n -0000026015 00000 n -0000026058 00000 n -0000026228 00000 n -0000026271 00000 n -0000026345 00000 n -0000026483 00000 n -0000026696 00000 n -0000026834 00000 n -0000027029 00000 n -0000027215 00000 n -0000027260 00000 n -0000027303 00000 n -0000027576 00000 n -0000027849 00000 n -0000035463 00000 n -0000035680 00000 n -0000037034 00000 n -0000037948 00000 n -0000045675 00000 n -0000045887 00000 n -0000047241 00000 n -0000048155 00000 n -0000050923 00000 n -0000051131 00000 n -0000052485 00000 n -0000053399 00000 n -0000057644 00000 n -0000057855 00000 n -0000059209 00000 n -0000060123 00000 n -0000063778 00000 n -0000063995 00000 n -0000064296 00000 n -trailer -<< /Size 52 -/Root 2 0 R -/Info 1 0 R ->> -startxref -65211 -%%EOF diff --git a/docs/metaData/Fields.adoc b/docs/metaData/Fields.adoc index da4f0993..c8dfd11f 100644 --- a/docs/metaData/Fields.adoc +++ b/docs/metaData/Fields.adoc @@ -56,6 +56,37 @@ if the value in the field is longer than the `maxLength`, then one of the follow ---- +===== ValueRangeBehavior +Used on Numeric fields. Specifies min and/or max allowed values for the field. +For each of min and max, the following attributes can be set: + +* `minValue` / `maxValue` - the number that is the limit. +* `minAllowEqualTo` / `maxAllowEqualTo` - boolean (default true). Controls if < (>) or ≤ (≥). +* `minBehavior` / `maxBehavior` - enum of `ERROR` (default) or `CLIP`. +** If `ERROR`, then a value not within the range causes an error, and the value does not get stored. +** else if `CLIP`, then a value not within the range gets "clipped" to either be the min/max (if allowEqualTo), +or to the min/max plus/minus the clipAmount +* `minClipAmount` / `maxClipAmount` - Default 1. Used when behavior is `CLIP` (only applies when +not allowEqualTo). + +[source,java] +.Examples of using ValueRangeBehavior +---- + new QFieldMetaData("noOfShoes", QFieldType.INTEGER) + .withBehavior(new ValueRangeBehavior().withMinValue(0)); + + new QFieldMetaData("price", QFieldType.BIG_DECIMAL) + .withBehavior(new ValueRangeBehavior() + // set the min value to be >= 0, and an error if an input is < 0. + .withMinValue(BigDecimal.ZERO) + .withMinAllowEqualTo(true) + .withMinBehavior(ERROR) + // set the max value to be < 100 - but effectively, clip larger values to 99.99 + // here we use the .withMax() method that takes 4 params vs. calling 4 .withMax*() methods. + .withMax(new BigDecimal("100.00"), false, CLIP, new BigDecimal("0.01")) + ); +---- + ===== DynamicDefaultValueBehavior Used to set a dynamic default value to a field when it is being inserted or updated. For example, instead of having a hard-coded `defaultValue` specified in the field meta-data, diff --git a/docs/metaData/MetaDataProduction.adoc b/docs/metaData/MetaDataProduction.adoc index da872485..353d77ff 100644 --- a/docs/metaData/MetaDataProduction.adoc +++ b/docs/metaData/MetaDataProduction.adoc @@ -288,33 +288,49 @@ all of the fields in you table in two places (the entity and the table meta-data If you are using `QRecordEntity` classes that correspond to your tables, then you can take advantage of some additional annotations on those classes, to produce more related meta-data objects associated with those tables. The point of this is to eliminate boilerplate, and simplify / speed up the process of getting a new table -built and deployed in your application, with some bells and whistles added. +built and deployed in your application. + +Furthermore, the case can be made that it is beneficial to keep the meta-data definition for a table as close +as possible to the entity that corresponds to the table. This enables modifications to the table (e.g., adding +a new field/column) to only require edits in one java source file, rather than necessarily requiring edits +in two files. === @QMetaDataProducingEntity -This is an annotation to go on a QRecordEntity class, which you would like to be -processed by `MetaDataProducerHelper`, to automatically produce some meta-data -objects. Specifically supports: +This is an annotation meant to be placed on a `QRecordEntity` subclass, which you would like to be +processed by an invocation of `MetaDataProducerHelper`, to automatically produce some meta-data +objects. -* Making a possible-value-source out of the table. +This annotation supports: + +* Creating table meta-data for the corresponding record entity table. Enabled by setting `produceTableMetaData=true`. +** One may customize the table meta data that is produced automatically by supplying a class that extends +`MetaDataCustomizerInterface` in the annotation attribute `tableMetaDataCustomizer`. +** In addition to (or as an alternative to) the per-table `MetaDataCustomizerInterface` that can be specified +in `@QMetaDataProducingEntity.tableMetaDataCustomzier`, when an application calls +`MetaDataProducerHelper.processAllMetaDataProducersInPackage`, an additional `MetaDataCustomizerInterface` can be +given, to apply a common set of adjustments to all tales being generated by the call. +* Making a possible-value-source out of the table. Enabled by setting `producePossibleValueSource=true`. * Processing child tables to create joins and childRecordList widgets === @ChildTable This is an annotation used as a value that goes inside a `@QMetadataProducingEntity` annotation, to define child-tables, e.g., for producing joins and childRecordList widgets related to the table defined in the entity class. -=== @ChildJoin +==== @ChildJoin This is an annotation used as a value inside a `@ChildTable` inside a `@QMetadataProducingEntity` annotation, to control the generation of a `QJoinMetaData`, as a `ONE_TO_MANY` type join from the table represented by the annotated entity, to the table referenced in the `@ChildTable` annotation. -=== @ChildRecordListWidget +==== @ChildRecordListWidget This is an annotation used as a value that goes inside a `@QMetadataProducingEntity` annotation, to control the generation of a QWidgetMetaData - for a ChildRecordList widget. [source,java] -.QRecordEntity with meta-data producing annotations +.QRecordEntity with meta-data producing annotations and a table MetaDataCustomizer ---- @QMetaDataProducingEntity( + produceTableMetaData = true, + tableMetaDataCustomizer = MyTable.TableMetaDataCustomizer.class, producePossibleValueSource = true, childTables = { @ChildTable( @@ -326,13 +342,47 @@ the generation of a QWidgetMetaData - for a ChildRecordList widget. public class MyTable extends QRecordEntity { public static final String TABLE_NAME = "myTable"; - // class body left as exercise for reader + + public static class TableMetaDataCustomizer implements MetaDataCustomizerInterface + { + @Override + public QTableMetaData customizeMetaData(QInstance qInstance, QTableMetaData table) throws QException + { + String childJoinName = QJoinMetaData.makeInferredJoinName(TABLE_NAME, MyChildTable.TABLE_NAME); + + table + .withUniqueKey(new UniqueKey("name")) + .withIcon(new QIcon().withName("table_bar")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("name") + + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, + List.of("id", "name"))) + // todo additional sections for other fields + .withSection(new QFieldSection("children", new QIcon().withName("account_tree"), Tier.T2) + .withWidgetName(childJoinName)) + + .withExposedJoin(new ExposedJoin() + .withLabel("Children") + .withJoinPath(List.of(childJoinName)) + .withJoinTable(MyChildTable.TABLE_NAME)); + + return (table); + } + } + + @QField(isEditable = false, isPrimaryKey = true) + private Integer id; + + // remaining fields, constructors, getters & setters left as an exercise for the reader and/or the IDE } ---- The class given in the example above, if processed by the `MetaDataProducerHelper`, would add the following meta-data objects to your `QInstance`: +* A `QTableMetaData` named `myTable`, with all fields annotated as `@QField` from the `QRecordEntity` class, +and with additional attributes as set in the `TableMetaDataCustomizer` inner class. * A `QPossibleValueSource` named `myTable`, of type `TABLE`, with `myTable` as its backing table. * A `QJoinMetaData` named `myTableJoinMyChildTable`, as a `ONE_TO_MANY` type, between those two tables. * A `QWidgetMetaData` named `myTableJoinMyChildTable`, as a `CHILD_RECORD_LIST` type, that will show a list of diff --git a/docs/metaData/Processes.adoc b/docs/metaData/Processes.adoc index 7985fb73..789bd6d6 100644 --- a/docs/metaData/Processes.adoc +++ b/docs/metaData/Processes.adoc @@ -38,6 +38,13 @@ See {link-permissionRules} for details. *** 1) by a single call to `.withStepList(List)`, which internally adds each step into the `steps` map. *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map. ** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map. +* `stepFlow` - *enum, default LINEAR* - specifies the the flow-control logic between steps. Possible values are: +** `LINEAR` - steps are executed in-order, through the `stepList`. +A backend step _can_ customize the `nextStepName` or re-order the `stepList`, if needed. +In a frontend step, a user may be given the option to go _back_ to a previous step as well. +** `STATE_MACHINE` - steps are executed as a Fine State Machine, starting with the first step in `stepList`, +but then proceeding based on the `nextStepName` specified by the previous step. +Thus allowing much more flexible flows. * `schedule` - *<>* - set up the process to run automatically on the specified schedule. See below for details. * `minInputRecords` - *Integer* - #not used...# @@ -67,6 +74,11 @@ For processes with a user-interface, they must define one or more "screens" in t * `formFields` - *List of String* - list of field names used by the screen as form-inputs. * `viewFields` - *List of String* - list of field names used by the screen as visible outputs. * `recordListFields` - *List of String* - list of field names used by the screen in a record listing. +* `format` - *Optional String* - directive for a frontend to use specialized formatting for the display of the process. +** Consult frontend documentation for supported values and their meanings. +* `backStepName` - *Optional String* - For processes using `LINEAR` flow, if this value is given, +then the frontend should offer a control that the user can take (e.g., a button) to move back to an +earlier step in the process. ==== QFrontendComponentMetaData @@ -90,10 +102,13 @@ Expects a process value named `html`. Expects process values named `downloadFileName` and `serverFilePath`. ** `GOOGLE_DRIVE_SELECT_FOLDER` - Special form that presents a UI from Google Drive, where the user can select a folder (e.g., as a target for uploading files in a subsequent backend step). ** `BULK_EDIT_FORM` - For use by the standard QQQ Bulk Edit process. +** `BULK_LOAD_FILE_MAPPING_FORM`, `BULK_LOAD_VALUE_MAPPING_FORM`, or `BULK_LOAD_PROFILE_FORM` - For use by the standard QQQ Bulk Load process. ** `VALIDATION_REVIEW_SCREEN` - For use by the QQQ Streamed ETL With Frontend process family of processes. Displays a component prompting the user to run full validation or to skip it, or, if full validation has been ran, then showing the results of that validation. ** `PROCESS_SUMMARY_RESULTS` - For use by the QQQ Streamed ETL With Frontend process family of processes. Displays the summary results of running the process. +** `WIDGET` - Render a QQQ Widget. +Requires that `widgetName` be given as a value for the component. ** `RECORD_LIST` - _Deprecated. Showed a grid with a list of records as populated by the process._ * `values` - *Map of String → Serializable* - Key=value pairs, with different expectations based on the component's `type`. @@ -116,6 +131,27 @@ It can be used, however, for example, to cause a `defaultValue` to be applied to It can also be used to cause the process to throw an error, if a field is marked as `isRequired`, but a value is not present. ** `recordListMetaData` - *RecordListMetaData object* - _Not used at this time._ +==== QStateMachineStep + +Processes that use `flow = STATE_MACHINE` should use process steps of type `QStateMachineStep`. + +A common pattern seen in state-machine processes, is that they will present a frontend-step to a user, +then always run a given backend-step in response to that screen which the user submitted. +Inside that backend-step, custom application logic will determine the next state to go to, +which is typically another frontend-step (which would then submit data to its corresponding backend-step, +and continue the FSM). + +To help facilitate this pattern, factory methods exist on `QStateMachineStep`, +for constructing the commonly-expected types of state-machine steps: + +* `frontendThenBackend(name, frontendStep, backendStep)` - for the frontend-then-backend pattern described above. +* `backendOnly(name, backendStep)` - for a state that only has a backend step. +This might be useful as a “reset” step, to run before restarting a state-loop. +* `frontendOnly(name, frontendStep)` - for a state that only has a frontend step, +which would always be followed by another state, which must be specified as the `defaultNextStepName` +on the `QStateMachineStep`. + + ==== BasepullConfiguration A "Basepull" process is a common pattern where an application needs to perform some action on all new (or updated) records from a particular data source. @@ -218,12 +254,10 @@ But for some cases, doing page-level transactions can reduce long-transactions a * `withSchedule(QScheduleMetaData schedule)` - Add a <> to the process. [#_custom_process_flow] -==== Custom Process Flow -As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process -will execute each of its steps in-order, as defined in the `stepList` property. -However, a Backend Step can customize this flow #todo - write more clearly here... - -There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow: +==== How to customize a Linear process flow +As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, +(with `flow = LINEAR`) a process will execute each of its steps in-order, as defined in the `stepList` property. +However, a Backend Step can customize this flow as follows: * `RunBackendStepOutput.setOverrideLastStepName(String stepName)` ** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next. @@ -239,7 +273,7 @@ does need to be found in the new `stepNameList` - otherwise, the framework will for figuring out where to go next. [source,java] -.Example of a defining process that can use a flexible flow: +.Example of a defining process that can use a customized linear flow: ---- // for a case like this, it would be recommended to define all step names in constants: public final static String STEP_START = "start"; @@ -324,4 +358,21 @@ public static class StartStep implements BackendStep } ---- +[#_process_back] +==== How to allow a process to go back + +The simplest option to allow a process to present a "Back" button to users, +thus allowing them to move backward through a process +(e.g., from a review screen back to an earlier input screen), is to set the property `backStepName` +on a `QFrontendStepMetaData`. + +If the step that is executed after the user hits "Back" is a backend step, then within that +step, `runBackendStepInput.getIsStepBack()` will return `true` (but ONLY within that first step after +the user hits "Back"). It may be necessary within individual processes to be aware that the user +has chosen to go back, to reset certain values in the process's state. + +Alternatively, if a frontend step's "Back" behavior needs to be dynamic (e.g., sometimes not available, +or sometimes targeting different steps in the process), then in a backend step that runs before the +frontend step, a call to `runBackendStepOutput.getProcessState().setBackStepName()` can be made, +to customize the value which would otherwise come from the `QFrontendStepMetaData`. diff --git a/pom.xml b/pom.xml index 68110a6b..bc43a583 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ qqq-backend-module-api qqq-backend-module-filesystem qqq-backend-module-rdbms + qqq-backend-module-sqlite qqq-backend-module-mongodb qqq-language-support-javascript qqq-openapi @@ -47,7 +48,7 @@ - 0.24.0-SNAPSHOT + 0.25.0-SNAPSHOT UTF-8 UTF-8 diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index c9a3c8f5..21723b83 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -100,7 +100,12 @@ org.dhatim fastexcel - 0.12.15 + 0.18.4 + + + org.dhatim + fastexcel-reader + 0.18.4 org.apache.poi @@ -112,6 +117,14 @@ poi-ooxml 5.2.5 + + + + commons-io + commons-io + 2.16.0 + + com.auth0 auth0 diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java index fa91e9ab..53947578 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; @@ -160,4 +161,18 @@ public interface RecordCustomizerUtilityInterface return (oldRecordMap); } + + /*************************************************************************** + ** + ***************************************************************************/ + static T getValueFromRecordOrOldRecord(String fieldName, QRecord record, Serializable primaryKey, Optional> oldRecordMap) + { + T value = (T) record.getValue(fieldName); + if(value == null && primaryKey != null && oldRecordMap.isPresent() && oldRecordMap.get().containsKey(primaryKey)) + { + value = (T) oldRecordMap.get().get(primaryKey).getValue(fieldName); + } + return value; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java index 3a7cfa41..486a11ea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java @@ -22,15 +22,19 @@ package com.kingsrook.qqq.backend.core.actions.customizers; +import java.io.Serializable; import java.util.List; +import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -47,7 +51,6 @@ public interface TableCustomizerInterface { QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class); - /******************************************************************************* ** custom actions to run after a query (or get!) takes place. ** @@ -77,8 +80,15 @@ public interface TableCustomizerInterface *******************************************************************************/ default List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException { - LOG.info("A default implementation of preInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); - return (records); + try + { + return (preInsertOrUpdate(insertInput, records, isPreview, Optional.empty())); + } + catch(NotImplementedHereException e) + { + LOG.info("A default implementation of preInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); + return (records); + } } @@ -104,8 +114,15 @@ public interface TableCustomizerInterface *******************************************************************************/ default List postInsert(InsertInput insertInput, List records) throws QException { - LOG.info("A default implementation of postInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); - return (records); + try + { + return (postInsertOrUpdate(insertInput, records, Optional.empty())); + } + catch(NotImplementedHereException e) + { + LOG.info("A default implementation of postInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); + return (records); + } } @@ -130,8 +147,15 @@ public interface TableCustomizerInterface *******************************************************************************/ default List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException { - LOG.info("A default implementation of preUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); - return (records); + try + { + return (preInsertOrUpdate(updateInput, records, isPreview, oldRecordList)); + } + catch(NotImplementedHereException e) + { + LOG.info("A default implementation of preUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); + return (records); + } } @@ -151,8 +175,15 @@ public interface TableCustomizerInterface *******************************************************************************/ default List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException { - LOG.info("A default implementation of postUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); - return (records); + try + { + return (postInsertOrUpdate(updateInput, records, oldRecordList)); + } + catch(NotImplementedHereException e) + { + LOG.info("A default implementation of postUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); + return (records); + } } @@ -199,4 +230,59 @@ public interface TableCustomizerInterface return (records); } + + /*************************************************************************** + ** Optional method to override in a customizer that does the same thing for + ** both preInsert & preUpdate. + ***************************************************************************/ + default List preInsertOrUpdate(AbstractActionInput input, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + throw NotImplementedHereException.instance; + } + + + /*************************************************************************** + ** Optional method to override in a customizer that does the same thing for + ** both postInsert & postUpdate. + ***************************************************************************/ + default List postInsertOrUpdate(AbstractActionInput input, List records, Optional> oldRecordList) throws QException + { + throw NotImplementedHereException.instance; + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default Optional> oldRecordListToMap(String primaryKeyField, Optional> oldRecordList) + { + if(oldRecordList.isPresent()) + { + return (Optional.of(CollectionUtils.listToMap(oldRecordList.get(), r -> r.getValue(primaryKeyField)))); + } + else + { + return (Optional.empty()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + class NotImplementedHereException extends QException + { + private static NotImplementedHereException instance = new NotImplementedHereException(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + private NotImplementedHereException() + { + super("Not implemented here"); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java index e8f654ed..a73fcb09 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java @@ -37,6 +37,8 @@ import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; @@ -51,12 +53,15 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.AbstractWidgetMetaDataBuilder; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -83,7 +88,9 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer .withIsCard(true) .withCodeReference(new QCodeReference(ChildRecordListRenderer.class)) .withType(WidgetType.CHILD_RECORD_LIST.getType()) - .withDefaultValue("joinName", join.getName()))); + .withDefaultValue("joinName", join.getName()) + .withValidatorPlugin(new ChildRecordListWidgetValidator()) + )); } @@ -168,6 +175,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName); return (this); } + } @@ -194,7 +202,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer } else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")) { - maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")); + maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().get("maxRows")); } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -299,6 +307,13 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer } } } + + if(widgetValues.containsKey("defaultValuesForNewChildRecordsFromParentFields")) + { + @SuppressWarnings("unchecked") + Map defaultValuesForNewChildRecordsFromParentFields = (Map) widgetValues.get("defaultValuesForNewChildRecordsFromParentFields"); + widgetData.setDefaultValuesForNewChildRecordsFromParentFields(defaultValuesForNewChildRecordsFromParentFields); + } } widgetData.setAllowRecordEdit(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("allowRecordEdit")))); @@ -313,4 +328,68 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer } } + + + /*************************************************************************** + ** + ***************************************************************************/ + private static class ChildRecordListWidgetValidator implements QInstanceValidatorPluginInterface + { + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void validate(QWidgetMetaDataInterface widgetMetaData, QInstance qInstance, QInstanceValidator qInstanceValidator) + { + String prefix = "Widget " + widgetMetaData.getName() + ": "; + + ////////////////////////////////// + // make sure join name is given // + ////////////////////////////////// + String joinName = ValueUtils.getValueAsString(CollectionUtils.nonNullMap(widgetMetaData.getDefaultValues()).get("joinName")); + if(qInstanceValidator.assertCondition(StringUtils.hasContent(joinName), prefix + "defaultValue for joinName must be given")) + { + /////////////////////////// + // make sure join exists // + /////////////////////////// + QJoinMetaData join = qInstance.getJoin(joinName); + if(qInstanceValidator.assertCondition(join != null, prefix + "No join named " + joinName + " exists in the instance")) + { + ////////////////////////////////////////////////////////////////////////////////// + // if there's a manageAssociationName, make sure the table has that association // + ////////////////////////////////////////////////////////////////////////////////// + String manageAssociationName = ValueUtils.getValueAsString(widgetMetaData.getDefaultValues().get("manageAssociationName")); + if(StringUtils.hasContent(manageAssociationName)) + { + validateAssociationName(prefix, manageAssociationName, join, qInstance, qInstanceValidator); + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void validateAssociationName(String prefix, String manageAssociationName, QJoinMetaData join, QInstance qInstance, QInstanceValidator qInstanceValidator) + { + /////////////////////////////////// + // make sure join's table exists // + /////////////////////////////////// + QTableMetaData table = qInstance.getTable(join.getLeftTable()); + if(table == null) + { + qInstanceValidator.getErrors().add(prefix + "Unable to validate manageAssociationName, as table [" + join.getLeftTable() + "] on left-side table of join [" + join.getName() + "] does not exist."); + } + else + { + if(CollectionUtils.nonNullList(table.getAssociations()).stream().noneMatch(a -> manageAssociationName.equals(a.getName()))) + { + qInstanceValidator.getErrors().add(prefix + "an association named [" + manageAssociationName + "] does not exist on table [" + join.getLeftTable() + "]"); + } + } + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java new file mode 100644 index 00000000..ffd6710e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java @@ -0,0 +1,251 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.AbstractWidgetMetaDataBuilder; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Generic widget to display a list of records. + ** + ** Note, closely related to (and copied from ChildRecordListRenderer. + ** opportunity to share more code with that in the future?? + *******************************************************************************/ +public class RecordListWidgetRenderer extends AbstractWidgetRenderer +{ + private static final QLogger LOG = QLogger.getLogger(RecordListWidgetRenderer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Builder widgetMetaDataBuilder(String widgetName) + { + return (new Builder(new QWidgetMetaData() + .withName(widgetName) + .withIsCard(true) + .withCodeReference(new QCodeReference(RecordListWidgetRenderer.class)) + .withType(WidgetType.CHILD_RECORD_LIST.getType()) + .withValidatorPlugin(new RecordListWidgetValidator()) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class Builder extends AbstractWidgetMetaDataBuilder + { + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Builder(QWidgetMetaData widgetMetaData) + { + super(widgetMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withLabel(String label) + { + widgetMetaData.setLabel(label); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withMaxRows(Integer maxRows) + { + widgetMetaData.withDefaultValue("maxRows", maxRows); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withTableName(String tableName) + { + widgetMetaData.withDefaultValue("tableName", tableName); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withFilter(QQueryFilter filter) + { + widgetMetaData.withDefaultValue("filter", filter); + return (this); + } + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + try + { + Integer maxRows = null; + if(StringUtils.hasContent(input.getQueryParams().get("maxRows"))) + { + maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows")); + } + else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")) + { + maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().get("maxRows")); + } + + QQueryFilter filter = ((QQueryFilter) input.getWidgetMetaData().getDefaultValues().get("filter")).clone(); + filter.interpretValues(new HashMap<>(input.getQueryParams()), FilterUseCase.DEFAULT); + filter.setLimit(maxRows); + + String tableName = ValueUtils.getValueAsString(input.getWidgetMetaData().getDefaultValues().get("tableName")); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(tableName); + queryInput.setShouldTranslatePossibleValues(true); + queryInput.setShouldGenerateDisplayValues(true); + queryInput.setFilter(filter); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + QValueFormatter.setBlobValuesToDownloadUrls(table, queryOutput.getRecords()); + + int totalRows = queryOutput.getRecords().size(); + if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) + { + ///////////////////////////////////////////////////////////////////////////////////// + // if the input said to only do some max, and the # of results we got is that max, // + // then do a count query, for displaying 1-n of // + ///////////////////////////////////////////////////////////////////////////////////// + CountInput countInput = new CountInput(); + countInput.setTableName(tableName); + countInput.setFilter(filter); + totalRows = new CountAction().execute(countInput).getCount(); + } + + String tablePath = QContext.getQInstance().getTablePath(tableName); + String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset())); + + ChildRecordListData widgetData = new ChildRecordListData(input.getQueryParams().get("widgetLabel"), queryOutput, table, tablePath, viewAllLink, totalRows); + + return (new RenderWidgetOutput(widgetData)); + } + catch(Exception e) + { + LOG.warn("Error rendering record list widget", e, logPair("widgetName", () -> input.getWidgetMetaData().getName())); + throw (e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static class RecordListWidgetValidator implements QInstanceValidatorPluginInterface + { + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void validate(QWidgetMetaDataInterface widgetMetaData, QInstance qInstance, QInstanceValidator qInstanceValidator) + { + String prefix = "Widget " + widgetMetaData.getName() + ": "; + + ////////////////////////////////////////////// + // make sure table name is given and exists // + ////////////////////////////////////////////// + QTableMetaData table = null; + String tableName = ValueUtils.getValueAsString(CollectionUtils.nonNullMap(widgetMetaData.getDefaultValues()).get("tableName")); + if(qInstanceValidator.assertCondition(StringUtils.hasContent(tableName), prefix + "defaultValue for tableName must be given")) + { + //////////////////////////// + // make sure table exists // + //////////////////////////// + table = qInstance.getTable(tableName); + qInstanceValidator.assertCondition(table != null, prefix + "No table named " + tableName + " exists in the instance"); + } + + //////////////////////////////////////////////////////////////////////////////////// + // make sure filter is given and is valid (only check that if table is given too) // + //////////////////////////////////////////////////////////////////////////////////// + QQueryFilter filter = ((QQueryFilter) widgetMetaData.getDefaultValues().get("filter")); + if(qInstanceValidator.assertCondition(filter != null, prefix + "defaultValue for filter must be given") && table != null) + { + qInstanceValidator.validateQueryFilter(qInstance, prefix, table, filter, null); + } + } + + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java index 119df559..bc0c205c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java @@ -36,6 +36,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -46,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -188,21 +191,40 @@ public class RunBackendStepAction { if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords())) { - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(inputMetaData.getRecordListMetaData().getTableName()); + QTableMetaData table = QContext.getQInstance().getTable(inputMetaData.getRecordListMetaData().getTableName()); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(table.getName()); - // todo - handle this being async (e.g., http) - // seems like it just needs to throw, breaking this flow, and to send a response to the frontend, directing it to prompt the user for the needed data - // then this step can re-run, hopefully with the needed data. - - QProcessCallback callback = runBackendStepInput.getCallback(); - if(callback == null) + ////////////////////////////////////////////////// + // look for record ids in the input data values // + ////////////////////////////////////////////////// + String recordIds = (String) runBackendStepInput.getValue("recordIds"); + if(recordIds == null) { - throw (new QUserFacingException("Missing input records.", - new QException("Function is missing input records, but no callback was present to request fields from a user"))); + recordIds = (String) runBackendStepInput.getValue("recordId"); } - queryInput.setFilter(callback.getQueryFilter()); + /////////////////////////////////////////////////////////// + // if records were found, add as criteria to query input // + /////////////////////////////////////////////////////////// + if(recordIds != null) + { + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds.split(",")))); + } + else + { + // todo - handle this being async (e.g., http) + // seems like it just needs to throw, breaking this flow, and to send a response to the frontend, directing it to prompt the user for the needed data + // then this step can re-run, hopefully with the needed data. + QProcessCallback callback = runBackendStepInput.getCallback(); + if(callback == null) + { + throw (new QUserFacingException("Missing input records.", + new QException("Function is missing input records, but no callback was present to request fields from a user"))); + } + + queryInput.setFilter(callback.getQueryFilter()); + } ////////////////////////////////////////////////////////////////////////////////////////// // if process has a max-no of records, set a limit on the process of that number plus 1 // @@ -210,7 +232,7 @@ public class RunBackendStepAction ////////////////////////////////////////////////////////////////////////////////////////// if(process.getMaxInputRecords() != null) { - if(callback.getQueryFilter() == null) + if(queryInput.getFilter() == null) { queryInput.setFilter(new QQueryFilter()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 481201e3..806b0f5c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -32,6 +32,7 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.NoCodeWidgetRenderer; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; @@ -53,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; @@ -63,6 +65,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; +import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface; import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider; import com.kingsrook.qqq.backend.core.state.StateProviderInterface; import com.kingsrook.qqq.backend.core.state.StateType; @@ -71,6 +74,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang.BooleanUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -87,12 +91,16 @@ public class RunProcessAction public static final String BASEPULL_TIMESTAMP_FIELD = "basepullTimestampField"; public static final String BASEPULL_CONFIGURATION = "basepullConfiguration"; + public static final String PROCESS_TRACER_CODE_REFERENCE_FIELD = "processTracerCodeReference"; + //////////////////////////////////////////////////////////////////////////////////////////////// // indicator that the timestamp field should be updated - e.g., the execute step is finished. // //////////////////////////////////////////////////////////////////////////////////////////////// public static final String BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD = "basepullReadyToUpdateTimestamp"; public static final String BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD = "basepullDidQueryUsingTimestamp"; + private ProcessTracerInterface processTracer; + /******************************************************************************* @@ -119,9 +127,17 @@ public class RunProcessAction } runProcessOutput.setProcessUUID(runProcessInput.getProcessUUID()); + traceStartOrResume(runProcessInput, process); + UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS); ProcessState processState = primeProcessState(runProcessInput, stateKey, process); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // these should always be clear when we're starting a run - so make sure they haven't leaked from previous // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + processState.clearNextStepName(); + processState.clearBackStepName(); + ///////////////////////////////////////////////////////// // if process is 'basepull' style, keep track of 'now' // ///////////////////////////////////////////////////////// @@ -160,6 +176,7 @@ public class RunProcessAction //////////////////////////////////////////////////////////// // upon exception (e.g., one thrown by a step), throw it. // //////////////////////////////////////////////////////////// + traceBreakOrFinish(runProcessInput, runProcessOutput, qe); throw (qe); } catch(Exception e) @@ -167,6 +184,7 @@ public class RunProcessAction //////////////////////////////////////////////////////////// // upon exception (e.g., one thrown by a step), throw it. // //////////////////////////////////////////////////////////// + traceBreakOrFinish(runProcessInput, runProcessOutput, e); throw (new QException("Error running process", e)); } finally @@ -177,6 +195,8 @@ public class RunProcessAction runProcessOutput.setProcessState(processState); } + traceBreakOrFinish(runProcessInput, runProcessOutput, null); + return (runProcessOutput); } @@ -188,14 +208,35 @@ public class RunProcessAction private void runLinearStepLoop(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws Exception { String lastStepName = runProcessInput.getStartAfterStep(); + String startAtStep = runProcessInput.getStartAtStep(); while(true) { /////////////////////////////////////////////////////////////////////////////////////////////////////// // always refresh the step list - as any step that runs can modify it (in the process state). // // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. // + // deal with if we were told, from the input, to start After a step, or start At a step. // /////////////////////////////////////////////////////////////////////////////////////////////////////// - List stepList = getAvailableStepList(processState, process, lastStepName); + List stepList; + if(startAtStep == null) + { + stepList = getAvailableStepList(processState, process, lastStepName, false); + } + else + { + stepList = getAvailableStepList(processState, process, startAtStep, true); + + /////////////////////////////////////////////////////////////////////////////////// + // clear this field - so after we run a step, we'll then loop in last-step mode. // + /////////////////////////////////////////////////////////////////////////////////// + startAtStep = null; + + /////////////////////////////////////////////////////////////////////////////////// + // if we're going to run a backend step now, let it see that this is a step-back // + /////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(true); + } + if(stepList.isEmpty()) { break; @@ -232,7 +273,18 @@ public class RunProcessAction ////////////////////////////////////////////////// throw (new QException("Unsure how to run a step of type: " + step.getClass().getName())); } + + //////////////////////////////////////////////////////////////////////////////////////// + // only let this value be set for the original back step - don't let it stick around. // + // if a process wants to keep track of this itself, it can, but in a different slot. // + //////////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(false); } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // in case we broke from the loop above (e.g., by going directly into a frontend step), once again make sure to lower this flag. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(false); } @@ -264,6 +316,12 @@ public class RunProcessAction processFrontendStepFieldDefaultValues(processState, step); processFrontendComponents(processState, step); processState.setNextStepName(step.getName()); + + if(StringUtils.hasContent(step.getBackStepName()) && processState.getBackStepName().isEmpty()) + { + processState.setBackStepName(step.getBackStepName()); + } + return LoopTodo.BREAK; } case SKIP -> @@ -317,6 +375,7 @@ public class RunProcessAction // else run the given lastStepName // ///////////////////////////////////// processState.clearNextStepName(); + processState.clearBackStepName(); step = process.getStep(lastStepName); if(step == null) { @@ -398,6 +457,7 @@ public class RunProcessAction // its sub-steps, or, to fall out of the loop and end the process. // ////////////////////////////////////////////////////////////////////////////////////////////////////// processState.clearNextStepName(); + processState.clearBackStepName(); runStateMachineStep(nextStepName.get(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1); return; } @@ -584,6 +644,7 @@ public class RunProcessAction runBackendStepInput.setCallback(runProcessInput.getCallback()); runBackendStepInput.setFrontendStepBehavior(runProcessInput.getFrontendStepBehavior()); runBackendStepInput.setAsyncJobCallback(runProcessInput.getAsyncJobCallback()); + runBackendStepInput.setProcessTracer(processTracer); runBackendStepInput.setTableName(process.getTableName()); if(!StringUtils.hasContent(runBackendStepInput.getTableName())) @@ -605,9 +666,13 @@ public class RunProcessAction runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY)); } + traceStepStart(runBackendStepInput); + RunBackendStepOutput runBackendStepOutput = new RunBackendStepAction().execute(runBackendStepInput); storeState(stateKey, runBackendStepOutput.getProcessState()); + traceStepFinish(runBackendStepInput, runBackendStepOutput); + if(runBackendStepOutput.getException() != null) { runProcessOutput.setException(runBackendStepOutput.getException()); @@ -621,8 +686,10 @@ public class RunProcessAction /******************************************************************************* ** Get the list of steps which are eligible to run. + ** + ** lastStep will be included in the list, or not, based on includeLastStep. *******************************************************************************/ - private List getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep) throws QException + static List getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep, boolean includeLastStep) throws QException { if(lastStep == null) { @@ -649,6 +716,10 @@ public class RunProcessAction if(stepName.equals(lastStep)) { foundLastStep = true; + if(includeLastStep) + { + validStepNames.add(stepName); + } } } return (stepNamesToSteps(process, validStepNames)); @@ -660,7 +731,7 @@ public class RunProcessAction /******************************************************************************* ** *******************************************************************************/ - private List stepNamesToSteps(QProcessMetaData process, List stepNames) throws QException + private static List stepNamesToSteps(QProcessMetaData process, List stepNames) throws QException { List result = new ArrayList<>(); @@ -744,13 +815,14 @@ public class RunProcessAction { QSession session = QContext.getQSession(); QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend()); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) { LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"); } else { - basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue()); + basepullKeyValue += "-" + session.getBackendVariants().get(variantTypeKey); } } @@ -879,4 +951,153 @@ public class RunProcessAction runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField()); runProcessInput.getValues().put(BASEPULL_CONFIGURATION, basepullConfiguration); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void setupProcessTracer(RunProcessInput runProcessInput, QProcessMetaData process) + { + try + { + if(process.getProcessTracerCodeReference() != null) + { + processTracer = QCodeLoader.getAdHoc(ProcessTracerInterface.class, process.getProcessTracerCodeReference()); + } + + Serializable processTracerCodeReference = runProcessInput.getValue(PROCESS_TRACER_CODE_REFERENCE_FIELD); + if(processTracerCodeReference != null) + { + if(processTracerCodeReference instanceof QCodeReference codeReference) + { + processTracer = QCodeLoader.getAdHoc(ProcessTracerInterface.class, codeReference); + } + } + } + catch(Exception e) + { + LOG.warn("Error setting up processTracer", e, logPair("processName", runProcessInput.getProcessName())); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void traceStartOrResume(RunProcessInput runProcessInput, QProcessMetaData process) + { + setupProcessTracer(runProcessInput, process); + + try + { + if(processTracer != null) + { + if(StringUtils.hasContent(runProcessInput.getStartAfterStep()) || StringUtils.hasContent(runProcessInput.getStartAtStep())) + { + processTracer.handleProcessResume(runProcessInput); + } + else + { + processTracer.handleProcessStart(runProcessInput); + } + } + } + catch(Exception e) + { + LOG.info("Error in traceStart", e, logPair("processName", runProcessInput.getProcessName())); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void traceBreakOrFinish(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException) + { + try + { + if(processTracer != null) + { + ProcessState processState = runProcessOutput.getProcessState(); + boolean isBreak = true; + + ///////////////////////////////////////////////////////////// + // if there's no next step, that means the process is done // + ///////////////////////////////////////////////////////////// + if(processState.getNextStepName().isEmpty()) + { + isBreak = false; + } + else + { + ///////////////////////////////////////////////////////////////// + // or if the next step is the last index, then we're also done // + ///////////////////////////////////////////////////////////////// + String nextStepName = processState.getNextStepName().get(); + int nextStepIndex = processState.getStepList().indexOf(nextStepName); + if(nextStepIndex == processState.getStepList().size() - 1) + { + isBreak = false; + } + } + + if(isBreak) + { + processTracer.handleProcessBreak(runProcessInput, runProcessOutput, processException); + } + else + { + processTracer.handleProcessFinish(runProcessInput, runProcessOutput, processException); + } + } + } + catch(Exception e) + { + LOG.info("Error in traceProcessFinish", e, logPair("processName", runProcessInput.getProcessName())); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void traceStepStart(RunBackendStepInput runBackendStepInput) + { + try + { + if(processTracer != null) + { + processTracer.handleStepStart(runBackendStepInput); + } + } + catch(Exception e) + { + LOG.info("Error in traceStepFinish", e, logPair("processName", runBackendStepInput.getProcessName()), logPair("stepName", runBackendStepInput.getStepName())); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void traceStepFinish(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + try + { + if(processTracer != null) + { + processTracer.handleStepFinish(runBackendStepInput, runBackendStepOutput); + } + } + catch(Exception e) + { + LOG.info("Error in traceStepFinish", e, logPair("processName", runBackendStepInput.getProcessName()), logPair("stepName", runBackendStepInput.getStepName())); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java index fac33e13..d36305f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java @@ -108,7 +108,7 @@ public class CsvExportStreamer implements ExportStreamerInterface } catch(Exception e) { - throw (new QReportingException("Error starting CSV report")); + throw (new QReportingException("Error starting CSV report", e)); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index 8acdf977..b1e28903 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -71,6 +71,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAndJoinTable; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; @@ -567,7 +568,7 @@ public class GenerateReportAction extends AbstractQActionFunction -1) - { - String joinTableName = fieldName.replaceAll("\\..*", ""); - String joinFieldName = fieldName.replaceAll(".*\\.", ""); - - QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName); - if(joinTable == null) - { - throw (new QException("Unrecognized join table name: " + joinTableName)); - } - - return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable); - } - else - { - return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable); - } - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -756,7 +731,7 @@ public class GenerateReportAction extends AbstractQActionFunction fields = new ArrayList<>(); for(String summaryFieldName : view.getSummaryFields()) { - FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName); + FieldAndJoinTable fieldAndJoinTable = FieldAndJoinTable.get(table, summaryFieldName); fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type? if so need table as input here } for(QReportField column : view.getColumns()) @@ -1208,27 +1183,4 @@ public class GenerateReportAction extends AbstractQActionFunction primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get()); - ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE); + ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE, deleteInput.getTransaction()); /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-delete customizer, if there is one // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 24c6e983..c98e31f3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -238,6 +238,11 @@ public class GetAction *******************************************************************************/ public static QRecord execute(String tableName, Serializable primaryKey) throws QException { + if(primaryKey instanceof QQueryFilter) + { + LOG.warn("Unexpected use of QQueryFilter instead of primary key in GetAction call"); + } + GetAction getAction = new GetAction(); GetInput getInput = new GetInput(tableName).withPrimaryKey(primaryKey); return getAction.executeForRecord(getInput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 75b17a22..5ce2ce38 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -110,6 +111,12 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); - if(preInsertCustomizer.isPresent()) - { - runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); - } + Optional preInsertCustomizer = didAlreadyRunCustomizer ? Optional.empty() : QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); setDefaultValuesInRecords(table, insertInput.getRecords()); @@ -258,7 +263,7 @@ public class InsertAction extends AbstractQActionFunction queryStatJoinTableList = new ArrayList<>(); for(String joinTableName : queryStat.getJoinTableNames()) { - queryStatJoinTableList.add(new QueryStatJoinTable().withQqqTableId(getQQQTableId(joinTableName))); + queryStatJoinTableList.add(new QueryStatJoinTable().withQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, joinTableName))); } queryStat.setQueryStatJoinTableList(queryStatJoinTableList); } @@ -460,7 +454,7 @@ public class QueryStatManager String[] parts = fieldName.split("\\."); if(parts.length > 1) { - queryStatCriteriaField.setQqqTableId(getQQQTableId(parts[0])); + queryStatCriteriaField.setQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, parts[0])); queryStatCriteriaField.setName(parts[1]); } } @@ -498,7 +492,7 @@ public class QueryStatManager String[] parts = fieldName.split("\\."); if(parts.length > 1) { - queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0])); + queryStatOrderByField.setQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, parts[0])); queryStatOrderByField.setName(parts[1]); } } @@ -512,44 +506,6 @@ public class QueryStatManager } } } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static Integer getQQQTableId(String tableName) throws QException - { - ///////////////////////////// - // look in the cache table // - ///////////////////////////// - GetInput getInput = new GetInput(); - getInput.setTableName(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME); - getInput.setUniqueKey(MapBuilder.of("name", tableName)); - GetOutput getOutput = new GetAction().execute(getInput); - - //////////////////////// - // upon cache miss... // - //////////////////////// - if(getOutput.getRecord() == null) - { - /////////////////////////////////////////////////////// - // insert the record (into the table, not the cache) // - /////////////////////////////////////////////////////// - QTableMetaData tableMetaData = getInstance().qInstance.getTable(tableName); - InsertInput insertInput = new InsertInput(); - insertInput.setTableName(QQQTable.TABLE_NAME); - insertInput.setRecords(List.of(new QRecord().withValue("name", tableName).withValue("label", tableMetaData.getLabel()))); - InsertOutput insertOutput = new InsertAction().execute(insertInput); - - /////////////////////////////////// - // repeat the get from the cache // - /////////////////////////////////// - getOutput = new GetAction().execute(getInput); - } - - return getOutput.getRecord().getValueInteger("id"); - } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java index dcc730d9..ac0280d7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -83,7 +84,7 @@ public class ValidateRecordSecurityLockHelper /******************************************************************************* ** *******************************************************************************/ - public static void validateSecurityFields(QTableMetaData table, List records, Action action) throws QException + public static void validateSecurityFields(QTableMetaData table, List records, Action action, QBackendTransaction transaction) throws QException { MultiRecordSecurityLock locksToCheck = getRecordSecurityLocks(table, action); if(locksToCheck == null || CollectionUtils.nullSafeIsEmpty(locksToCheck.getLocks())) @@ -101,7 +102,7 @@ public class ValidateRecordSecurityLockHelper // actually check lock values // //////////////////////////////// Map errorRecords = new HashMap<>(); - evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys); + evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys, transaction); ///////////////////////////////// // propagate errors to records // @@ -141,7 +142,7 @@ public class ValidateRecordSecurityLockHelper ** BUT - WRITE locks - in their case, we read the record no matter what, and in ** here we need to verify we have a key that allows us to WRITE the record. *******************************************************************************/ - private static void evaluateRecordLocks(QTableMetaData table, List records, Action action, RecordSecurityLock recordSecurityLock, Map errorRecords, List treePosition, Map madeUpPrimaryKeys) throws QException + private static void evaluateRecordLocks(QTableMetaData table, List records, Action action, RecordSecurityLock recordSecurityLock, Map errorRecords, List treePosition, Map madeUpPrimaryKeys, QBackendTransaction transaction) throws QException { if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock) { @@ -152,7 +153,7 @@ public class ValidateRecordSecurityLockHelper for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks())) { treePosition.add(i); - evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys); + evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys, transaction); treePosition.remove(treePosition.size() - 1); i++; } @@ -225,6 +226,7 @@ public class ValidateRecordSecurityLockHelper // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) // //////////////////////////////////////////////////////////////////////////////////////////////// QueryInput queryInput = new QueryInput(); + queryInput.setTransaction(transaction); queryInput.setTableName(leftMostJoin.getLeftTable()); QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); queryInput.setFilter(filter); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java index 6e0c068a..f9142f02 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java @@ -104,10 +104,21 @@ public class RenderTemplateAction extends AbstractQActionFunction context, String code) throws QException + { + return (renderVelocity(context, code)); + } + + + /******************************************************************************* ** Most convenient static wrapper to render a Velocity template. *******************************************************************************/ - public static String renderVelocity(AbstractActionInput parentActionInput, Map context, String code) throws QException + public static String renderVelocity(Map context, String code) throws QException { return (render(TemplateType.VELOCITY, context, code)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java index 09e786c5..d32e4e79 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; @@ -74,6 +75,31 @@ public interface QCustomPossibleValueProvider } + /*************************************************************************** + ** meant to be protected (but interface...) - for a custom PVS implementation + ** to complete its search (e.g., after it generates the list of PVS objects, + ** let this method do the filtering). + ***************************************************************************/ + default List> completeCustomPVSSearch(SearchPossibleValueSourceInput input, List> possibleValues) + { + SearchPossibleValueSourceAction.PreparedSearchPossibleValueSourceInput preparedInput = SearchPossibleValueSourceAction.prepareSearchPossibleValueSourceInput(input); + + List> rs = new ArrayList<>(); + + for(QPossibleValue possibleValue : possibleValues) + { + if(possibleValue != null && SearchPossibleValueSourceAction.doesPossibleValueMatchSearchInput(possibleValue, preparedInput)) + { + rs.add(possibleValue); + } + } + + rs.sort(Comparator.nullsLast(Comparator.comparing((QPossibleValue pv) -> pv.getLabel()))); + + return (rs); + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 3c695cb7..876b76ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -28,6 +28,7 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -484,6 +486,8 @@ public class QValueFormatter String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT)); String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION)); + Boolean downloadUrlDynamic = ValueUtils.getValueAsBoolean(adornmentValues.get(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC)); + for(QRecord record : records) { if(!doesFieldHaveValue(field, record)) @@ -491,6 +495,11 @@ public class QValueFormatter continue; } + if(BooleanUtils.isTrue(downloadUrlDynamic)) + { + continue; + } + Serializable primaryKey = record.getValue(table.getPrimaryKeyField()); String fileName = null; @@ -508,7 +517,7 @@ public class QValueFormatter { @SuppressWarnings("unchecked") // instance validation should make this safe! List fileNameFormatFields = (List) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS); - List values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList(); + List values = CollectionUtils.nullSafeHasContents(fileNameFormatFields) ? fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList() : Collections.emptyList(); fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values); } } @@ -531,7 +540,7 @@ public class QValueFormatter //////////////////////////////////////////////////////////////////////////////////////////////// // if field type is blob OR if there's a supplemental process or code-ref that needs to run - // - // then update its value to be a callback-url that'll give access to the bytes to download // + // then update its value to be a callback-url that'll give access to the bytes to download. // // implied here is that a String value (w/o supplemental code/proc) has its value stay as a // // URL, which is where the file is directly downloaded from. And in the case of a String // // with code-to-run, then the code should run, followed by a redirect to the value URL. // @@ -540,7 +549,7 @@ public class QValueFormatter || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE) || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME)) { - record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName); + record.setValue(field.getName(), AdornmentType.FileDownloadValues.makeFieldDownloadUrl(table.getName(), primaryKey, field.getName(), fileName)); } record.setDisplayValue(field.getName(), fileName); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index 9cb17a0c..6416ff50 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -26,8 +26,13 @@ import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -47,10 +52,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; -import org.apache.commons.lang.NotImplementedException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -61,6 +65,9 @@ public class SearchPossibleValueSourceAction { private static final QLogger LOG = QLogger.getLogger(SearchPossibleValueSourceAction.class); + private static final Set warnedAboutUnexpectedValueField = Collections.synchronizedSet(new HashSet<>()); + private static final Set warnedAboutUnexpectedNoOfFieldsToSearchByLabel = Collections.synchronizedSet(new HashSet<>()); + private QPossibleValueTranslator possibleValueTranslator; @@ -101,47 +108,54 @@ public class SearchPossibleValueSourceAction + /*************************************************************************** + ** record to store "computed" values as part of a possible-value search - + ** e.g., ids type-convered, and lower-cased labels. + ***************************************************************************/ + public record PreparedSearchPossibleValueSourceInput(Collection inputIdsAsCorrectType, Collection lowerCaseLabels, String searchTerm) {} + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static PreparedSearchPossibleValueSourceInput prepareSearchPossibleValueSourceInput(SearchPossibleValueSourceInput input) + { + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(input.getPossibleValueSourceName()); + List inputIdsAsCorrectType = convertInputIdsToPossibleValueSourceIdType(possibleValueSource, input.getIdList()); + + Set lowerCaseLabels = null; + if(input.getLabelList() != null) + { + lowerCaseLabels = input.getLabelList().stream() + .filter(Objects::nonNull) + .map(l -> l.toLowerCase()) + .collect(Collectors.toSet()); + } + + return (new PreparedSearchPossibleValueSourceInput(inputIdsAsCorrectType, lowerCaseLabels, input.getSearchTerm())); + } + + + /******************************************************************************* ** *******************************************************************************/ private SearchPossibleValueSourceOutput searchPossibleValueEnum(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) { + PreparedSearchPossibleValueSourceInput preparedSearchPossibleValueSourceInput = prepareSearchPossibleValueSourceInput(input); + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceOutput(); List matchingIds = new ArrayList<>(); - List inputIdsAsCorrectType = convertInputIdsToEnumIdType(possibleValueSource, input.getIdList()); - for(QPossibleValue possibleValue : possibleValueSource.getEnumValues()) { - boolean match = false; - - if(input.getIdList() != null) - { - if(inputIdsAsCorrectType.contains(possibleValue.getId())) - { - match = true; - } - } - else - { - if(StringUtils.hasContent(input.getSearchTerm())) - { - match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.getSearchTerm().toLowerCase()) - || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase())); - } - else - { - match = true; - } - } + boolean match = doesPossibleValueMatchSearchInput(possibleValue, preparedSearchPossibleValueSourceInput); if(match) { - matchingIds.add((Serializable) possibleValue.getId()); + matchingIds.add(possibleValue.getId()); } - - // todo - skip & limit? - // todo - default filter } List> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, matchingIds); @@ -152,37 +166,95 @@ public class SearchPossibleValueSourceAction + /*************************************************************************** + ** + ***************************************************************************/ + public static boolean doesPossibleValueMatchSearchInput(QPossibleValue possibleValue, PreparedSearchPossibleValueSourceInput input) + { + boolean match = false; + + if(input.inputIdsAsCorrectType() != null) + { + if(input.inputIdsAsCorrectType().contains(possibleValue.getId())) + { + match = true; + } + } + else if(input.lowerCaseLabels() != null) + { + if(input.lowerCaseLabels().contains(possibleValue.getLabel().toLowerCase())) + { + match = true; + } + } + else + { + if(StringUtils.hasContent(input.searchTerm())) + { + match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.searchTerm().toLowerCase()) + || possibleValue.getLabel().toLowerCase().startsWith(input.searchTerm().toLowerCase())); + } + else + { + match = true; + } + } + return match; + } + + + /******************************************************************************* ** The input list of ids might come through as a type that isn't the same as ** the type of the ids in the enum (e.g., strings from a frontend, integers - ** in an enum). So, this method looks at the first id in the enum, and then - ** maps all the inputIds to be of the same type. + ** in an enum). So, this method type-converts them. *******************************************************************************/ - private List convertInputIdsToEnumIdType(QPossibleValueSource possibleValueSource, List inputIdList) + private static List convertInputIdsToPossibleValueSourceIdType(QPossibleValueSource possibleValueSource, List inputIdList) { List rs = new ArrayList<>(); - if(CollectionUtils.nullSafeIsEmpty(inputIdList)) + + if(inputIdList == null) + { + return (null); + } + else if(inputIdList.isEmpty()) { return (rs); } - Object anIdFromTheEnum = possibleValueSource.getEnumValues().get(0).getId(); + QFieldType type = possibleValueSource.getIdType(); - if(anIdFromTheEnum instanceof Integer) + for(Serializable inputId : inputIdList) { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsInteger(id))); - } - else if(anIdFromTheEnum instanceof String) - { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsString(id))); - } - else if(anIdFromTheEnum instanceof Boolean) - { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsBoolean(id))); - } - else - { - LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName()); + Object properlyTypedId = null; + try + { + if(type.equals(QFieldType.INTEGER)) + { + properlyTypedId = ValueUtils.getValueAsInteger(inputId); + } + else if(type.isStringLike()) + { + properlyTypedId = ValueUtils.getValueAsString(inputId); + } + else if(type.equals(QFieldType.BOOLEAN)) + { + properlyTypedId = ValueUtils.getValueAsBoolean(inputId); + } + else + { + LOG.warn("Unexpected type [" + type + "] for ids in enum: " + possibleValueSource.getName()); + } + } + catch(Exception e) + { + LOG.debug("Error converting possible value id to expected id type", e, logPair("value", inputId)); + } + + if(properlyTypedId != null) + { + rs.add(properlyTypedId); + } } return (rs); @@ -209,6 +281,53 @@ public class SearchPossibleValueSourceAction { queryFilter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, input.getIdList())); } + else if(input.getLabelList() != null) + { + List fieldNames = new ArrayList<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // the 'value fields' will either be 'id' or 'label' (which means, use the fields from the tableMetaData's label fields) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(String valueField : possibleValueSource.getValueFields()) + { + if("id".equals(valueField)) + { + fieldNames.add(table.getPrimaryKeyField()); + } + else if("label".equals(valueField)) + { + if(table.getRecordLabelFields() != null) + { + fieldNames.addAll(table.getRecordLabelFields()); + } + } + else + { + String message = "Unexpected valueField defined in possibleValueSource when searching possibleValueSource by label (required: 'id' or 'label')"; + if(!warnedAboutUnexpectedValueField.contains(possibleValueSource.getName())) + { + LOG.warn(message, logPair("valueField", valueField), logPair("possibleValueSource", possibleValueSource.getName())); + warnedAboutUnexpectedValueField.add(possibleValueSource.getName()); + } + output.setWarning(message); + } + } + + if(fieldNames.size() == 1) + { + queryFilter.addCriteria(new QFilterCriteria(fieldNames.get(0), QCriteriaOperator.IN, input.getLabelList())); + } + else + { + String message = "Unexpected number of fields found for searching possibleValueSource by label (required: 1, found: " + fieldNames.size() + ")"; + if(!warnedAboutUnexpectedNoOfFieldsToSearchByLabel.contains(possibleValueSource.getName())) + { + LOG.warn(message); + warnedAboutUnexpectedNoOfFieldsToSearchByLabel.add(possibleValueSource.getName()); + } + output.setWarning(message); + } + } else { String searchTerm = input.getSearchTerm(); @@ -269,8 +388,8 @@ public class SearchPossibleValueSourceAction queryFilter = input.getDefaultQueryFilter(); } - // todo - skip & limit as params - queryFilter.setLimit(250); + queryFilter.setLimit(input.getLimit()); + queryFilter.setSkip(input.getSkip()); queryFilter.setOrderBys(possibleValueSource.getOrderByFields()); @@ -288,7 +407,7 @@ public class SearchPossibleValueSourceAction fieldName = table.getPrimaryKeyField(); } - List ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList(); + List ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList(); List> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, ids); output.setResults(qPossibleValues); @@ -301,7 +420,7 @@ public class SearchPossibleValueSourceAction ** *******************************************************************************/ @SuppressWarnings({ "rawtypes", "unchecked" }) - private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) + private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) throws QException { try { @@ -314,11 +433,10 @@ public class SearchPossibleValueSourceAction } catch(Exception e) { - // LOG.warn("Error sending [" + value + "] for field [" + field + "] through custom code for PVS [" + field.getPossibleValueSourceName() + "]", e); + String message = "Error searching custom possible value source [" + input.getPossibleValueSourceName() + "]"; + LOG.warn(message, e); + throw (new QException(message, e)); } - - throw new NotImplementedException("Not impleemnted"); - // return (null); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 69dcb786..f0b82ab0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -23,7 +23,11 @@ package com.kingsrook.qqq.backend.core.instances; import java.io.Serializable; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -31,21 +35,27 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType.FileUploadAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; @@ -54,6 +64,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; @@ -75,13 +86,20 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileUploadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.utils.ClassPathUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -107,6 +125,8 @@ public class QInstanceEnricher ////////////////////////////////////////////////////////////////////////////////////////////////// private static final Map labelMappings = new LinkedHashMap<>(); + private static ListingHash, QInstanceEnricherPluginInterface> enricherPlugins = new ListingHash<>(); + /******************************************************************************* @@ -168,6 +188,7 @@ public class QInstanceEnricher } enrichJoins(); + enrichInstance(); ////////////////////////////////////////////////////////////////////////////// // if the instance DOES have 1 or more scheduler, but no schedulable types, // @@ -184,6 +205,16 @@ public class QInstanceEnricher + /*************************************************************************** + ** + ***************************************************************************/ + private void enrichInstance() + { + runPlugins(QInstance.class, qInstance, qInstance); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -248,6 +279,14 @@ public class QInstanceEnricher } } } + + /////////////////////////////////////////// + // run plugins on joins if there are any // + /////////////////////////////////////////// + for(QJoinMetaData join : qInstance.getJoins().values()) + { + runPlugins(QJoinMetaData.class, join, qInstance); + } } catch(Exception e) { @@ -263,6 +302,7 @@ public class QInstanceEnricher private void enrichWidget(QWidgetMetaDataInterface widgetMetaData) { enrichPermissionRules(widgetMetaData); + runPlugins(QWidgetMetaDataInterface.class, widgetMetaData, qInstance); } @@ -273,6 +313,7 @@ public class QInstanceEnricher private void enrichBackend(QBackendMetaData qBackendMetaData) { qBackendMetaData.enrich(); + runPlugins(QBackendMetaData.class, qBackendMetaData, qInstance); } @@ -327,6 +368,7 @@ public class QInstanceEnricher enrichPermissionRules(table); enrichAuditRules(table); + runPlugins(QTableMetaData.class, table, qInstance); } @@ -417,6 +459,7 @@ public class QInstanceEnricher } enrichPermissionRules(process); + runPlugins(QProcessMetaData.class, process, qInstance); } @@ -538,6 +581,8 @@ public class QInstanceEnricher field.withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE); } } + + runPlugins(QFieldMetaData.class, field, qInstance); } @@ -609,6 +654,7 @@ public class QInstanceEnricher ensureAppSectionMembersAreAppChildren(app); enrichPermissionRules(app); + runPlugins(QAppMetaData.class, app, qInstance); } @@ -756,6 +802,7 @@ public class QInstanceEnricher } enrichPermissionRules(report); + runPlugins(QReportMetaData.class, report, qInstance); } @@ -847,7 +894,7 @@ public class QInstanceEnricher /******************************************************************************* ** *******************************************************************************/ - private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) + public void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) { Map values = new HashMap<>(); values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); @@ -859,6 +906,7 @@ public class QInstanceEnricher values ) .withName(processName) + .withIcon(new QIcon().withName("library_add")) .withLabel(table.getLabel() + " Bulk Insert") .withTableName(table.getName()) .withIsHidden(true) @@ -889,18 +937,76 @@ public class QInstanceEnricher .map(QFieldMetaData::getLabel) .collect(Collectors.joining(", ")); + QBackendStepMetaData prepareFileUploadStep = new QBackendStepMetaData() + .withName("prepareFileUpload") + .withCode(new QCodeReference(BulkInsertPrepareFileUploadStep.class)); + QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() .withName("upload") .withLabel("Upload File") - .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("previewText", "file upload instructions") - .withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText)) + .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB) + .withFieldAdornment(FileUploadAdornment.newFieldAdornment() + .withValue(FileUploadAdornment.formatDragAndDrop()) + .withValue(FileUploadAdornment.widthFull())) + .withLabel(table.getLabel() + " File") + .withIsRequired(true)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML)) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); - process.addStep(0, uploadScreen); - process.getFrontendStep("review").setRecordListFields(editableFields); + QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData() + .withName("prepareFileMapping") + .withCode(new QCodeReference(BulkInsertPrepareFileMappingStep.class)); + + QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() + .withName("fileMapping") + .withLabel("File Mapping") + .withBackStepName("prepareFileUpload") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)) + .withFormField(new QFieldMetaData("hasHeaderRow", QFieldType.BOOLEAN)) + .withFormField(new QFieldMetaData("layout", QFieldType.STRING)); // is actually PVS, but, this field is only added to help support helpContent, so :shrug: + + QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() + .withName("receiveFileMapping") + .withCode(new QCodeReference(BulkInsertReceiveFileMappingStep.class)); + + QBackendStepMetaData prepareValueMappingStep = new QBackendStepMetaData() + .withName("prepareValueMapping") + .withCode(new QCodeReference(BulkInsertPrepareValueMappingStep.class)); + + QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData() + .withName("valueMapping") + .withLabel("Value Mapping") + .withBackStepName("prepareFileMapping") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM)); + + QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData() + .withName("receiveValueMapping") + .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); + + int i = 0; + process.addStep(i++, prepareFileUploadStep); + process.addStep(i++, uploadScreen); + + process.addStep(i++, prepareFileMappingStep); + process.addStep(i++, fileMappingScreen); + process.addStep(i++, receiveFileMappingStep); + + process.addStep(i++, prepareValueMappingStep); + process.addStep(i++, valueMappingScreen); + process.addStep(i++, receiveValueMappingStep); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields); + + ////////////////////////////////////////////////////////////////////////////////////////// + // put the bulk-load profile form (e.g., for saving it) on the review & result screens) // + ////////////////////////////////////////////////////////////////////////////////////////// + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .withBackStepName("prepareFileMapping") + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT) + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + qInstance.addProcess(process); } @@ -1295,6 +1401,137 @@ public class QInstanceEnricher } } + if(possibleValueSource.getIdType() == null) + { + QTableMetaData table = qInstance.getTable(possibleValueSource.getTableName()); + if(table != null) + { + String primaryKeyField = table.getPrimaryKeyField(); + QFieldMetaData primaryKeyFieldMetaData = table.getFields().get(primaryKeyField); + if(primaryKeyFieldMetaData != null) + { + possibleValueSource.setIdType(primaryKeyFieldMetaData.getType()); + } + } + } + } + else if(QPossibleValueSourceType.ENUM.equals(possibleValueSource.getType())) + { + if(possibleValueSource.getIdType() == null) + { + if(CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues())) + { + Object id = possibleValueSource.getEnumValues().get(0).getId(); + try + { + possibleValueSource.setIdType(QFieldType.fromClass(id.getClass())); + } + catch(Exception e) + { + LOG.warn("Error enriching possible value source with idType based on first enum value", e, logPair("possibleValueSource", possibleValueSource.getName()), logPair("id", id)); + } + } + } + } + else if(QPossibleValueSourceType.CUSTOM.equals(possibleValueSource.getType())) + { + if(possibleValueSource.getIdType() == null) + { + try + { + QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource); + + Method getPossibleValueMethod = customPossibleValueProvider.getClass().getDeclaredMethod("getPossibleValue", Serializable.class); + Type returnType = getPossibleValueMethod.getGenericReturnType(); + Type idType = ((ParameterizedType) returnType).getActualTypeArguments()[0]; + + if(idType instanceof Class c) + { + possibleValueSource.setIdType(QFieldType.fromClass(c)); + } + } + catch(Exception e) + { + LOG.warn("Error enriching possible value source with idType based on first custom value", e, logPair("possibleValueSource", possibleValueSource.getName())); + } + } + } + + runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void addEnricherPlugin(QInstanceEnricherPluginInterface plugin) + { + Optional enrichMethod = Arrays.stream(plugin.getClass().getDeclaredMethods()) + .filter(m -> m.getName().equals("enrich") + && m.getParameterCount() == 2 + && !m.getParameterTypes()[0].equals(Object.class) + && m.getParameterTypes()[1].equals(QInstance.class) + ).findFirst(); + + if(enrichMethod.isPresent()) + { + Class parameterType = enrichMethod.get().getParameterTypes()[0]; + enricherPlugins.add(parameterType, plugin); + } + else + { + LOG.warn("Could not find enrich method on enricher plugin [" + plugin.getClass().getName() + "] (to infer type being enriched) - this plugin will not be used."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void removeAllEnricherPlugins() + { + enricherPlugins.clear(); + } + + + + /*************************************************************************** + ** scan the classpath for classes in the specified package name which + ** implement the QInstanceEnricherPluginInterface - any found get added + ***************************************************************************/ + public static void discoverAndAddPluginsInPackage(String packageName) throws QException + { + try + { + for(Class aClass : ClassPathUtils.getClassesInPackage(packageName)) + { + if(QInstanceEnricherPluginInterface.class.isAssignableFrom(aClass)) + { + QInstanceEnricherPluginInterface plugin = (QInstanceEnricherPluginInterface) aClass.getConstructor().newInstance(); + addEnricherPlugin(plugin); + } + } + } + catch(Exception e) + { + throw (new QException("Error discovering and adding enricher plugins in package [" + packageName + "]", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runPlugins(Class c, T t, QInstance qInstance) + { + for(QInstanceEnricherPluginInterface plugin : CollectionUtils.nonNullList(enricherPlugins.get(c))) + { + @SuppressWarnings("unchecked") + QInstanceEnricherPluginInterface castedPlugin = (QInstanceEnricherPluginInterface) plugin; + castedPlugin.enrich(t, qInstance); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java index 02376ff3..e46fb80a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java @@ -30,7 +30,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -111,7 +110,7 @@ public class QInstanceHelpContentManager } else { - LOG.info("Discarding help content with key that does not contain name:value format", logPair("key", key), logPair("id", record.getValue("id"))); + LOG.info("Discarding help content with key-part that does not contain name:value format", logPair("key", key), logPair("part", part), logPair("id", record.getValue("id"))); } } @@ -150,19 +149,19 @@ public class QInstanceHelpContentManager /////////////////////////////////////////////////////////////////////////////////// if(StringUtils.hasContent(tableName)) { - processHelpContentForTable(key, tableName, sectionName, fieldName, slotName, roles, helpContent); + processHelpContentForTable(qInstance, key, tableName, sectionName, fieldName, slotName, roles, helpContent); } else if(StringUtils.hasContent(processName)) { - processHelpContentForProcess(key, processName, fieldName, stepName, roles, helpContent); + processHelpContentForProcess(qInstance, key, processName, fieldName, stepName, roles, helpContent); } else if(StringUtils.hasContent(widgetName)) { - processHelpContentForWidget(key, widgetName, slotName, roles, helpContent); + processHelpContentForWidget(qInstance, key, widgetName, slotName, roles, helpContent); } else if(nameValuePairs.containsKey("instanceLevel")) { - processHelpContentForInstance(key, slotName, roles, helpContent); + processHelpContentForInstance(qInstance, key, slotName, roles, helpContent); } } catch(Exception e) @@ -176,9 +175,9 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForTable(String key, String tableName, String sectionName, String fieldName, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForTable(QInstance qInstance, String key, String tableName, String sectionName, String fieldName, String slotName, Set roles, QHelpContent helpContent) { - QTableMetaData table = QContext.getQInstance().getTable(tableName); + QTableMetaData table = qInstance.getTable(tableName); if(table == null) { LOG.info("Unrecognized table in help content", logPair("key", key)); @@ -246,9 +245,30 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForProcess(String key, String processName, String fieldName, String stepName, Set roles, QHelpContent helpContent) + private static void processHelpContentForProcess(QInstance qInstance, String key, String processName, String fieldName, String stepName, Set roles, QHelpContent helpContent) { - QProcessMetaData process = QContext.getQInstance().getProcess(processName); + if(processName.startsWith("*") && processName.length() > 1) + { + boolean anyMatched = false; + String subName = processName.substring(1); + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getName().endsWith(subName)) + { + anyMatched = true; + processHelpContentForProcess(qInstance, key, process.getName(), fieldName, stepName, roles, helpContent); + } + } + + if(!anyMatched) + { + LOG.info("Wildcard process name did not match any processes in help content", logPair("key", key)); + } + + return; + } + + QProcessMetaData process = qInstance.getProcess(processName); if(process == null) { LOG.info("Unrecognized process in help content", logPair("key", key)); @@ -306,9 +326,9 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForWidget(String key, String widgetName, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForWidget(QInstance qInstance, String key, String widgetName, String slotName, Set roles, QHelpContent helpContent) { - QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName); + QWidgetMetaDataInterface widget = qInstance.getWidget(widgetName); if(!StringUtils.hasContent(slotName)) { LOG.info("Missing slot name in help content", logPair("key", key)); @@ -335,7 +355,7 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForInstance(String key, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForInstance(QInstance qInstance, String key, String slotName, Set roles, QHelpContent helpContent) { if(!StringUtils.hasContent(slotName)) { @@ -345,11 +365,11 @@ public class QInstanceHelpContentManager { if(helpContent != null) { - QContext.getQInstance().withHelpContent(slotName, helpContent); + qInstance.withHelpContent(slotName, helpContent); } else { - QContext.getQInstance().removeHelpContent(slotName, roles); + qInstance.removeHelpContent(slotName, roles); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 4d5e8c51..b123858e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -37,7 +37,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TimeZone; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -108,12 +110,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.Automatio import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; +import org.apache.commons.lang.BooleanUtils; import org.quartz.CronExpression; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -136,6 +142,8 @@ public class QInstanceValidator private static ListingHash, QInstanceValidatorPluginInterface> validatorPlugins = new ListingHash<>(); + private JoinGraph joinGraph = null; + private List errors = new ArrayList<>(); @@ -163,8 +171,7 @@ public class QInstanceValidator // the enricher will build a join graph (if there are any joins). we'd like to only do that // // once, during the enrichment/validation work, so, capture it, and store it back in the instance. // ///////////////////////////////////////////////////////////////////////////////////////////////////// - JoinGraph joinGraph = null; - long start = System.currentTimeMillis(); + long start = System.currentTimeMillis(); try { ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -173,7 +180,7 @@ public class QInstanceValidator // TODO - possible point of customization (use a different enricher, or none, or pass it options). QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance); qInstanceEnricher.enrich(); - joinGraph = qInstanceEnricher.getJoinGraph(); + this.joinGraph = qInstanceEnricher.getJoinGraph(); } catch(Exception e) { @@ -373,8 +380,8 @@ public class QInstanceValidator assertCondition(join.getType() != null, "Missing type for join: " + joinName); assertCondition(CollectionUtils.nullSafeHasContents(join.getJoinOns()), "Missing joinOns for join: " + joinName); - boolean leftTableExists = assertCondition(qInstance.getTable(join.getLeftTable()) != null, "Left-table name " + join.getLeftTable() + " join " + joinName + " is not a defined table in this instance."); - boolean rightTableExists = assertCondition(qInstance.getTable(join.getRightTable()) != null, "Right-table name " + join.getRightTable() + " join " + joinName + " is not a defined table in this instance."); + boolean leftTableExists = assertCondition(qInstance.getTable(join.getLeftTable()) != null, "Left-table name " + join.getLeftTable() + " in join " + joinName + " is not a defined table in this instance."); + boolean rightTableExists = assertCondition(qInstance.getTable(join.getRightTable()) != null, "Right-table name " + join.getRightTable() + " in join " + joinName + " is not a defined table in this instance."); for(JoinOn joinOn : CollectionUtils.nonNullList(join.getJoinOns())) { @@ -543,6 +550,60 @@ public class QInstanceValidator { assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); + /////////////////////// + // validate variants // + /////////////////////// + BackendVariantsConfig backendVariantsConfig = backend.getBackendVariantsConfig(); + if(BooleanUtils.isTrue(backend.getUsesVariants())) + { + if(assertCondition(backendVariantsConfig != null, "Missing backendVariantsConfig in backend [" + backendName + "] which is marked as usesVariants")) + { + assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]"); + + String optionsTableName = backendVariantsConfig.getOptionsTableName(); + QTableMetaData optionsTable = qInstance.getTable(optionsTableName); + if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]")) + { + if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]")) + { + QQueryFilter optionsFilter = backendVariantsConfig.getOptionsFilter(); + if(optionsFilter != null) + { + validateQueryFilter(qInstance, "optionsFilter in backendVariantsConfig in backend [" + backendName + "]: ", optionsTable, optionsFilter, null); + } + } + } + + Map backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap(); + if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]")) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // only validate field names in the backendSettingSourceFieldNameMap if there is NOT a variantRecordSupplier // + // (the idea being, that the supplier might be building a record with fieldNames that aren't in the table... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(optionsTable != null && backendVariantsConfig.getVariantRecordLookupFunction() == null) + { + for(Map.Entry entry : backendSettingSourceFieldNameMap.entrySet()) + { + assertCondition(optionsTable.getFields().containsKey(entry.getValue()), "Unrecognized fieldName [" + entry.getValue() + "] in backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]"); + } + } + } + + if(backendVariantsConfig.getVariantRecordLookupFunction() != null) + { + validateSimpleCodeReference("VariantRecordSupplier in backendVariantsConfig in backend [" + backendName + "]: ", backendVariantsConfig.getVariantRecordLookupFunction(), UnsafeFunction.class, Function.class); + } + } + } + else + { + assertCondition(backendVariantsConfig == null, "Should not have a backendVariantsConfig in backend [" + backendName + "] which is not marked as usesVariants"); + } + + /////////////////////////////////////////// + // let the backend do its own validation // + /////////////////////////////////////////// backend.performValidation(this); runPlugins(QBackendMetaData.class, backend, qInstance); @@ -577,7 +638,7 @@ public class QInstanceValidator private void validateAuthentication(QInstance qInstance) { QAuthenticationMetaData authentication = qInstance.getAuthentication(); - if(authentication != null) + if(assertCondition(authentication != null, "Authentication MetaData must be defined.")) { if(authentication.getCustomizer() != null) { @@ -780,7 +841,7 @@ public class QInstanceValidator { if(assertCondition(StringUtils.hasContent(association.getName()), "missing a name for an Association on table " + table.getName())) { - String messageSuffix = " for Association " + association.getName() + " on table " + table.getName(); + String messageSuffix = " for Association " + association.getName() + " on table " + table.getName(); boolean recognizedTable = false; if(assertCondition(StringUtils.hasContent(association.getAssociatedTableName()), "missing associatedTableName" + messageSuffix)) { @@ -988,7 +1049,15 @@ public class QInstanceValidator @SuppressWarnings("unchecked") Class> behaviorClass = (Class>) fieldBehavior.getClass(); - errors.addAll(fieldBehavior.validateBehaviorConfiguration(table, field)); + List behaviorErrors = fieldBehavior.validateBehaviorConfiguration(table, field); + if(behaviorErrors != null) + { + String prefixMinusTrailingSpace = prefix.replaceFirst(" *$", ""); + for(String behaviorError : behaviorErrors) + { + errors.add(prefixMinusTrailingSpace + ": " + behaviorClass.getSimpleName() + ": " + behaviorError); + } + } if(!fieldBehavior.allowMultipleBehaviorsOfThisType()) { @@ -1348,7 +1417,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(customizerInstance != null && tableCustomizer.getExpectedType() != null) { - assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance); + assertObjectCanBeCasted(prefix, customizerInstance, tableCustomizer.getExpectedType()); } } } @@ -1360,18 +1429,31 @@ public class QInstanceValidator /******************************************************************************* ** Make sure that a given object can be casted to an expected type. *******************************************************************************/ - private T assertObjectCanBeCasted(String errorPrefix, Class expectedType, Object object) + private void assertObjectCanBeCasted(String errorPrefix, Object object, Class... anyOfExpectedClasses) { - T castedObject = null; - try + for(Class expectedClass : anyOfExpectedClasses) { - castedObject = expectedType.cast(object); + try + { + expectedClass.cast(object); + return; + } + catch(ClassCastException e) + { + ///////////////////////////////////// + // try next type (if there is one) // + ///////////////////////////////////// + } } - catch(ClassCastException e) + + if(anyOfExpectedClasses.length == 1) { - errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType); + errors.add(errorPrefix + "CodeReference is not of the expected type: " + anyOfExpectedClasses[0]); + } + else + { + errors.add(errorPrefix + "CodeReference is not any of the expected types: " + Arrays.stream(anyOfExpectedClasses).map(c -> c.getName()).collect(Collectors.joining(", "))); } - return castedObject; } @@ -1608,12 +1690,12 @@ public class QInstanceValidator for(QFieldMetaData field : process.getInputFields()) { - validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName()); + validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName() + " "); } for(QFieldMetaData field : process.getOutputFields()) { - validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName()); + validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName() + " "); } if(process.getCancelStep() != null) @@ -1824,7 +1906,7 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List queryJoins) + public void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List queryJoins) { for(QFilterCriteria criterion : CollectionUtils.nonNullList(queryFilter.getCriteria())) { @@ -1868,7 +1950,8 @@ public class QInstanceValidator { if(fieldName.contains(".")) { - String fieldNameAfterDot = fieldName.substring(fieldName.lastIndexOf(".") + 1); + String fieldNameAfterDot = fieldName.substring(fieldName.lastIndexOf(".") + 1); + String tableNameBeforeDot = fieldName.substring(0, fieldName.lastIndexOf(".")); if(CollectionUtils.nullSafeHasContents(queryJoins)) { @@ -1892,11 +1975,32 @@ public class QInstanceValidator } else { - errors.add("QInstanceValidator does not yet support finding a field that looks like a join field, but isn't associated with a query."); - return (true); - // todo! for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values()) - // { - // } + if(this.joinGraph != null) + { + Set joinConnections = joinGraph.getJoinConnections(table.getName()); + for(JoinGraph.JoinConnectionList joinConnectionList : joinConnections) + { + JoinGraph.JoinConnection joinConnection = joinConnectionList.list().get(joinConnectionList.list().size() - 1); + if(tableNameBeforeDot.equals(joinConnection.joinTable())) + { + QTableMetaData joinTable = qInstance.getTable(tableNameBeforeDot); + if(joinTable.getFields().containsKey(fieldNameAfterDot)) + { + ///////////////////////// + // mmm, looks valid... // + ///////////////////////// + return (true); + } + } + } + } + + ////////////////////////////////////////////////////////////////////////////////////// + // todo - not sure how vulnerable we are to ongoing issues here... // + // idea: let a filter (or any object?) be opted out of validation, some version of // + // a static map of objects we can check at the top of various validate methods... // + ////////////////////////////////////////////////////////////////////////////////////// + errors.add("Failed to find field named: " + fieldName); } } } @@ -2000,6 +2104,11 @@ public class QInstanceValidator } } + if(widget.getValidatorPlugin() != null) + { + widget.getValidatorPlugin().validate(widget, qInstance, this); + } + runPlugins(QWidgetMetaDataInterface.class, widget, qInstance); } ); @@ -2099,6 +2208,8 @@ public class QInstanceValidator default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType()); } + assertCondition(possibleValueSource.getIdType() != null, "possibleValueSource " + name + " is missing its idType."); + runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance); } } @@ -2108,7 +2219,8 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class expectedClass) + @SafeVarargs + private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class... anyOfExpectedClasses) { if(!preAssertionsForCodeReference(codeReference, prefix)) { @@ -2136,7 +2248,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(classInstance != null) { - assertObjectCanBeCasted(prefix, expectedClass, classInstance); + assertObjectCanBeCasted(prefix, classInstance, anyOfExpectedClasses); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/enrichment/plugins/QInstanceEnricherPluginInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/enrichment/plugins/QInstanceEnricherPluginInterface.java new file mode 100644 index 00000000..30b07f61 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/enrichment/plugins/QInstanceEnricherPluginInterface.java @@ -0,0 +1,40 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.enrichment.plugins; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; + + +/******************************************************************************* + ** Interface for additional / optional enrichment to be done on q instance members. + ** Some may be provided by QQQ - others can be defined by applications. + *******************************************************************************/ +public interface QInstanceEnricherPluginInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void enrich(T object, QInstance qInstance); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java index 83b8785c..35b09c45 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java @@ -326,6 +326,20 @@ public class AuditSingleInput implements Serializable + /******************************************************************************* + ** Fluent setter for details + *******************************************************************************/ + public AuditSingleInput withDetailMessages(List details) + { + for(String detail : details) + { + addDetail(message); + } + return (this); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java index ad7a0827..c6d07011 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java @@ -40,6 +40,8 @@ public class ProcessState implements Serializable private Map values = new HashMap<>(); private List stepList = new ArrayList<>(); private Optional nextStepName = Optional.empty(); + private Optional backStepName = Optional.empty(); + private boolean isStepBack = false; private ProcessMetaDataAdjustment processMetaDataAdjustment = null; @@ -122,6 +124,39 @@ public class ProcessState implements Serializable + /******************************************************************************* + ** Getter for backStepName + ** + *******************************************************************************/ + public Optional getBackStepName() + { + return backStepName; + } + + + + /******************************************************************************* + ** Setter for backStepName + ** + *******************************************************************************/ + public void setBackStepName(String backStepName) + { + this.backStepName = Optional.of(backStepName); + } + + + + /******************************************************************************* + ** clear out the value of backStepName (set the Optional to empty) + ** + *******************************************************************************/ + public void clearBackStepName() + { + this.backStepName = Optional.empty(); + } + + + /******************************************************************************* ** Getter for stepList ** @@ -176,4 +211,35 @@ public class ProcessState implements Serializable } + + /******************************************************************************* + ** Getter for isStepBack + *******************************************************************************/ + public boolean getIsStepBack() + { + return (this.isStepBack); + } + + + + /******************************************************************************* + ** Setter for isStepBack + *******************************************************************************/ + public void setIsStepBack(boolean isStepBack) + { + this.isStepBack = isStepBack; + } + + + + /******************************************************************************* + ** Fluent setter for isStepBack + *******************************************************************************/ + public ProcessState withIsStepBack(boolean isStepBack) + { + this.isStepBack = isStepBack; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java index 98697f80..d4974a07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -53,6 +53,7 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface ////////////////////////////////////////////////////////////////////////// private ArrayList primaryKeys; + private ArrayList bulletsOfText; /******************************************************************************* @@ -497,4 +498,35 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface return (this); } + + /******************************************************************************* + ** Getter for bulletsOfText + *******************************************************************************/ + public ArrayList getBulletsOfText() + { + return (this.bulletsOfText); + } + + + + /******************************************************************************* + ** Setter for bulletsOfText + *******************************************************************************/ + public void setBulletsOfText(ArrayList bulletsOfText) + { + this.bulletsOfText = bulletsOfText; + } + + + + /******************************************************************************* + ** Fluent setter for bulletsOfText + *******************************************************************************/ + public ProcessSummaryLine withBulletsOfText(ArrayList bulletsOfText) + { + this.bulletsOfText = bulletsOfText; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index bfaad833..ca066eb3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -25,19 +25,28 @@ package com.kingsrook.qqq.backend.core.model.actions.processes; import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface; +import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerMessage; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -46,6 +55,8 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class RunBackendStepInput extends AbstractActionInput { + private static final QLogger LOG = QLogger.getLogger(RunBackendStepInput.class); + private ProcessState processState; private String processName; private String tableName; @@ -55,12 +66,13 @@ public class RunBackendStepInput extends AbstractActionInput private RunProcessInput.FrontendStepBehavior frontendStepBehavior; private Instant basepullLastRunTime; + private ProcessTracerInterface processTracer; + //////////////////////////////////////////////////////////////////////////// // note - new fields should generally be added in method: cloneFieldsInto // //////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** *******************************************************************************/ @@ -96,6 +108,7 @@ public class RunBackendStepInput extends AbstractActionInput target.setAsyncJobCallback(getAsyncJobCallback()); target.setFrontendStepBehavior(getFrontendStepBehavior()); target.setValues(getValues()); + target.setProcessTracer(getProcessTracer().orElse(null)); } @@ -238,6 +251,26 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Getter for records converted to entities of a given type. + ** + *******************************************************************************/ + public List getRecordsAsEntities(Class entityClass) throws QException + { + List rs = new ArrayList<>(); + + /////////////////////////////////////////////////////////////////////////////////// + // note - important to call getRecords here, which is overwritten in subclasses! // + /////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : getRecords()) + { + rs.add(QRecordEntity.fromQRecord(entityClass, record)); + } + return (rs); + } + + + /******************************************************************************* ** Setter for records ** @@ -419,6 +452,17 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Accessor for processState's isStepBack attribute + ** + *******************************************************************************/ + public boolean getIsStepBack() + { + return processState.getIsStepBack(); + } + + + /******************************************************************************* ** Accessor for processState - protected, because we generally want to access ** its members through wrapper methods, we think @@ -524,4 +568,64 @@ public class RunBackendStepInput extends AbstractActionInput return (this); } + + + /******************************************************************************* + ** Setter for processTracer + *******************************************************************************/ + public void setProcessTracer(ProcessTracerInterface processTracer) + { + this.processTracer = processTracer; + } + + + + /******************************************************************************* + ** Fluent setter for processTracer + *******************************************************************************/ + public RunBackendStepInput withProcessTracer(ProcessTracerInterface processTracer) + { + this.processTracer = processTracer; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Optional getProcessTracer() + { + return Optional.ofNullable(processTracer); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void traceMessage(ProcessTracerMessage message) + { + if(processTracer != null && message != null) + { + try + { + processTracer.handleMessage(this, message); + } + catch(Exception e) + { + LOG.warn("Error tracing message", e, logPair("message", message)); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public QProcessMetaData getProcess() + { + return (QContext.getQInstance().getProcess(getProcessName())); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java index 7c89b98a..ba6b87c5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java @@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -258,7 +259,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial /******************************************************************************* - ** + ** add a record to the step output, e.g., for going through to the next step. *******************************************************************************/ public void addRecord(QRecord record) { @@ -271,6 +272,16 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial + /*************************************************************************** + ** add a RecordEntity to the step output, e.g., for going through to the next step. + ***************************************************************************/ + public void addRecordEntity(QRecordEntity recordEntity) + { + addRecord(recordEntity.toQRecord()); + } + + + /******************************************************************************* ** Getter for auditInputList *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java index a099caaf..c9d500f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java @@ -49,6 +49,7 @@ public class RunProcessInput extends AbstractActionInput private ProcessState processState; private FrontendStepBehavior frontendStepBehavior = FrontendStepBehavior.BREAK; private String startAfterStep; + private String startAtStep; private String processUUID; private AsyncJobCallback asyncJobCallback; @@ -451,4 +452,35 @@ public class RunProcessInput extends AbstractActionInput { return asyncJobCallback; } + + /******************************************************************************* + ** Getter for startAtStep + *******************************************************************************/ + public String getStartAtStep() + { + return (this.startAtStep); + } + + + + /******************************************************************************* + ** Setter for startAtStep + *******************************************************************************/ + public void setStartAtStep(String startAtStep) + { + this.startAtStep = startAtStep; + } + + + + /******************************************************************************* + ** Fluent setter for startAtStep + *******************************************************************************/ + public RunProcessInput withStartAtStep(String startAtStep) + { + this.startAtStep = startAtStep; + return (this); + } + + } \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaOption.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaOption.java new file mode 100644 index 00000000..e43cece8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaOption.java @@ -0,0 +1,31 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum CriteriaOption implements CriteriaOptionInterface +{ + CASE_INSENSITIVE; +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaOptionInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaOptionInterface.java new file mode 100644 index 00000000..44ffeb38 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaOptionInterface.java @@ -0,0 +1,30 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.query; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface CriteriaOptionInterface +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java index 9c88f19b..b5a17f84 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java @@ -26,8 +26,10 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization.QFilterCriteriaDeserializer; @@ -45,7 +47,7 @@ public class QFilterCriteria implements Serializable, Cloneable, QMetaDataObject { private static final QLogger LOG = QLogger.getLogger(QFilterCriteria.class); - private String fieldName; + private String fieldName; private QCriteriaOperator operator; private List values; @@ -54,6 +56,8 @@ public class QFilterCriteria implements Serializable, Cloneable, QMetaDataObject //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// private String otherFieldName; + private Set options = null; + /******************************************************************************* @@ -70,6 +74,13 @@ public class QFilterCriteria implements Serializable, Cloneable, QMetaDataObject clone.values = new ArrayList<>(); clone.values.addAll(values); } + + if(options != null) + { + clone.options = new HashSet<>(); + clone.options.addAll(options); + } + return clone; } catch(CloneNotSupportedException e) @@ -386,4 +397,78 @@ public class QFilterCriteria implements Serializable, Cloneable, QMetaDataObject return Objects.hash(fieldName, operator, values, otherFieldName); } + + + /******************************************************************************* + ** Getter for options + *******************************************************************************/ + public Set getOptions() + { + return (this.options); + } + + + + /******************************************************************************* + ** Setter for options + *******************************************************************************/ + public void setOptions(Set options) + { + this.options = options; + } + + + + /******************************************************************************* + ** Fluent setter for options + *******************************************************************************/ + public QFilterCriteria withOptions(Set options) + { + this.options = options; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public QFilterCriteria withOption(CriteriaOptionInterface option) + { + if(options == null) + { + options = new HashSet<>(); + } + options.add(option); + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public QFilterCriteria withoutOption(CriteriaOptionInterface option) + { + if(options != null) + { + options.remove(option); + } + return (this); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public boolean hasOption(CriteriaOptionInterface option) + { + if(options == null) + { + return (false); + } + + return (options.contains(option)); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index 97f094d9..95137359 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -854,4 +854,20 @@ public class QQueryFilter implements Serializable, Cloneable, QMetaDataObject } + /*************************************************************************** + ** + ***************************************************************************/ + public void applyCriteriaOptionToAllCriteria(CriteriaOptionInterface criteriaOption) + { + for(QFilterCriteria criteria : CollectionUtils.nonNullList(this.criteria)) + { + criteria.withOption(criteriaOption); + } + + for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters)) + { + subFilter.applyCriteriaOptionToAllCriteria(criteriaOption); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java index 526c7c78..b0fe962e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java @@ -22,13 +22,14 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.storage; +import java.io.Serializable; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; /******************************************************************************* ** Input for Storage actions. *******************************************************************************/ -public class StorageInput extends AbstractTableActionInput +public class StorageInput extends AbstractTableActionInput implements Serializable { private String reference; private String contentType; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java index 65b7c2bc..8976a176 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java @@ -38,9 +38,10 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen private QQueryFilter defaultQueryFilter; private String searchTerm; private List idList; + private List labelList; private Integer skip = 0; - private Integer limit = 100; + private Integer limit = 250; @@ -281,4 +282,35 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen this.limit = limit; return (this); } + + + /******************************************************************************* + ** Getter for labelList + *******************************************************************************/ + public List getLabelList() + { + return (this.labelList); + } + + + + /******************************************************************************* + ** Setter for labelList + *******************************************************************************/ + public void setLabelList(List labelList) + { + this.labelList = labelList; + } + + + + /******************************************************************************* + ** Fluent setter for labelList + *******************************************************************************/ + public SearchPossibleValueSourceInput withLabelList(List labelList) + { + this.labelList = labelList; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java index e7186614..e60ed262 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java @@ -35,6 +35,7 @@ public class SearchPossibleValueSourceOutput extends AbstractActionOutput { private List> results = new ArrayList<>(); + private String warning; /******************************************************************************* @@ -88,4 +89,35 @@ public class SearchPossibleValueSourceOutput extends AbstractActionOutput return (this); } + + /******************************************************************************* + ** Getter for warning + *******************************************************************************/ + public String getWarning() + { + return (this.warning); + } + + + + /******************************************************************************* + ** Setter for warning + *******************************************************************************/ + public void setWarning(String warning) + { + this.warning = warning; + } + + + + /******************************************************************************* + ** Fluent setter for warning + *******************************************************************************/ + public SearchPossibleValueSourceOutput withWarning(String warning) + { + this.warning = warning; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChildRecordListData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChildRecordListData.java index a6b3a6eb..a5992234 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChildRecordListData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChildRecordListData.java @@ -51,6 +51,7 @@ public class ChildRecordListData extends QWidgetData private boolean canAddChildRecord = false; private Map defaultValuesForNewChildRecords; private Set disabledFieldsForNewChildRecords; + private Map defaultValuesForNewChildRecordsFromParentFields; @@ -523,6 +524,37 @@ public class ChildRecordListData extends QWidgetData return (this); } + + + /******************************************************************************* + ** Getter for defaultValuesForNewChildRecordsFromParentFields + *******************************************************************************/ + public Map getDefaultValuesForNewChildRecordsFromParentFields() + { + return (this.defaultValuesForNewChildRecordsFromParentFields); + } + + + + /******************************************************************************* + ** Setter for defaultValuesForNewChildRecordsFromParentFields + *******************************************************************************/ + public void setDefaultValuesForNewChildRecordsFromParentFields(Map defaultValuesForNewChildRecordsFromParentFields) + { + this.defaultValuesForNewChildRecordsFromParentFields = defaultValuesForNewChildRecordsFromParentFields; + } + + + + /******************************************************************************* + ** Fluent setter for defaultValuesForNewChildRecordsFromParentFields + *******************************************************************************/ + public ChildRecordListData withDefaultValuesForNewChildRecordsFromParentFields(Map defaultValuesForNewChildRecordsFromParentFields) + { + this.defaultValuesForNewChildRecordsFromParentFields = defaultValuesForNewChildRecordsFromParentFields; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FilterAndColumnsSetupData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FilterAndColumnsSetupData.java index 10a562a0..f35afbb3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FilterAndColumnsSetupData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FilterAndColumnsSetupData.java @@ -34,8 +34,12 @@ public class FilterAndColumnsSetupData extends QWidgetData private String tableName; private Boolean allowVariables = false; private Boolean hideColumns = false; + private Boolean hidePreview = false; private List filterDefaultFieldNames; + private String filterFieldName = "queryFilterJson"; + private String columnFieldName = "columnsJson"; + /******************************************************************************* @@ -193,4 +197,97 @@ public class FilterAndColumnsSetupData extends QWidgetData return (this); } + + + /******************************************************************************* + ** Getter for hidePreview + *******************************************************************************/ + public Boolean getHidePreview() + { + return (this.hidePreview); + } + + + + /******************************************************************************* + ** Setter for hidePreview + *******************************************************************************/ + public void setHidePreview(Boolean hidePreview) + { + this.hidePreview = hidePreview; + } + + + + /******************************************************************************* + ** Fluent setter for hidePreview + *******************************************************************************/ + public FilterAndColumnsSetupData withHidePreview(Boolean hidePreview) + { + this.hidePreview = hidePreview; + return (this); + } + + + + /******************************************************************************* + ** Getter for filterFieldName + *******************************************************************************/ + public String getFilterFieldName() + { + return (this.filterFieldName); + } + + + + /******************************************************************************* + ** Setter for filterFieldName + *******************************************************************************/ + public void setFilterFieldName(String filterFieldName) + { + this.filterFieldName = filterFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for filterFieldName + *******************************************************************************/ + public FilterAndColumnsSetupData withFilterFieldName(String filterFieldName) + { + this.filterFieldName = filterFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnFieldName + *******************************************************************************/ + public String getColumnFieldName() + { + return (this.columnFieldName); + } + + + + /******************************************************************************* + ** Setter for columnFieldName + *******************************************************************************/ + public void setColumnFieldName(String columnFieldName) + { + this.columnFieldName = columnFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for columnFieldName + *******************************************************************************/ + public FilterAndColumnsSetupData withColumnFieldName(String columnFieldName) + { + this.columnFieldName = columnFieldName; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java index 13ead81c..f8fbfbe8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java @@ -89,6 +89,11 @@ public @interface QField *******************************************************************************/ int maxLength() default Integer.MAX_VALUE; + /******************************************************************************* + ** + *******************************************************************************/ + int gridColumns() default -1; + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index dffc5947..cc3b0f6c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -154,7 +154,7 @@ public class QRecord implements Serializable return (null); } - Map clone = new LinkedHashMap<>(); + Map clone = new LinkedHashMap<>(map.size()); for(Map.Entry entry : map.entrySet()) { Serializable value = entry.getValue(); @@ -246,6 +246,24 @@ public class QRecord implements Serializable } + /*************************************************************************** + ** copy all values from 'joinedRecord' into this record's values map, + ** prefixing field names with joinTableNam + "." + ***************************************************************************/ + public void addJoinedRecordValues(String joinTableName, QRecord joinedRecord) + { + if(joinedRecord == null) + { + return; + } + + for(Map.Entry entry : joinedRecord.getValues().entrySet()) + { + setValue(joinTableName + "." + entry.getKey(), entry.getValue()); + } + } + + /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java index e62757e1..c07446a0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java @@ -41,11 +41,14 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Function; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -61,6 +64,11 @@ public abstract class QRecordEntity private Map originalRecordValues; + //////////////////////////////////////////////////////////////////////////////// + // map of entity class names to QTableMetaData objects that they helped build // + //////////////////////////////////////////////////////////////////////////////// + private static Map tableReferences = new HashMap<>(); + /******************************************************************************* @@ -95,6 +103,19 @@ public abstract class QRecordEntity + /*************************************************************************** + ** register a mapping between an entity class and a table that it is associated with. + ***************************************************************************/ + public static void registerTable(Class entityClass, QTableMetaData table) + { + if(entityClass != null && table != null) + { + tableReferences.put(entityClass.getName(), table); + } + } + + + /******************************************************************************* ** Build an entity of this QRecord type from a QRecord ** @@ -176,7 +197,10 @@ public abstract class QRecordEntity /******************************************************************************* - ** Convert this entity to a QRecord. + ** Convert this entity to a QRecord. ALL fields in the entity will be set + ** in the QRecord. Note that, if you're using this for an input to the UpdateAction, + ** that this could cause values to be set to null, e.g., if you constructed + ** a entity from scratch, and didn't set all values in it!! ** *******************************************************************************/ public QRecord toQRecord() throws QRuntimeException @@ -190,25 +214,7 @@ public abstract class QRecordEntity qRecord.setValue(qRecordEntityField.getFieldName(), (Serializable) qRecordEntityField.getGetter().invoke(this)); } - for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass())) - { - @SuppressWarnings("unchecked") - List associatedEntities = (List) qRecordEntityAssociation.getGetter().invoke(this); - String associationName = qRecordEntityAssociation.getAssociationAnnotation().name(); - - if(associatedEntities != null) - { - ///////////////////////////////////////////////////////////////////////////////// - // do this so an empty list in the entity becomes an empty list in the QRecord // - ///////////////////////////////////////////////////////////////////////////////// - qRecord.withAssociatedRecords(associationName, new ArrayList<>()); - } - - for(QRecordEntity associatedEntity : CollectionUtils.nonNullList(associatedEntities)) - { - qRecord.withAssociatedRecord(associationName, associatedEntity.toQRecord()); - } - } + toQRecordProcessAssociations(qRecord, (entity) -> entity.toQRecord()); return (qRecord); } @@ -220,15 +226,65 @@ public abstract class QRecordEntity + /*************************************************************************** + * + ***************************************************************************/ + private void toQRecordProcessAssociations(QRecord outputRecord, Function toRecordFunction) throws Exception + { + for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass())) + { + @SuppressWarnings("unchecked") + List associatedEntities = (List) qRecordEntityAssociation.getGetter().invoke(this); + String associationName = qRecordEntityAssociation.getAssociationAnnotation().name(); + + if(associatedEntities != null) + { + outputRecord.withAssociatedRecords(associationName, new ArrayList<>()); + for(QRecordEntity associatedEntity : associatedEntities) + { + outputRecord.withAssociatedRecord(associationName, toRecordFunction.apply(associatedEntity)); + } + } + } + } + + + /******************************************************************************* - ** + ** Overload of toQRecordOnlyChangedFields that preserves original behavior of + ** that method, which is, to NOT includePrimaryKey *******************************************************************************/ + @Deprecated(since = "includePrimaryKey param was added") public QRecord toQRecordOnlyChangedFields() + { + return toQRecordOnlyChangedFields(false); + } + + + + /******************************************************************************* + ** Useful for the use-case of: + ** - fetch a QRecord (e.g., QueryAction or GetAction) + ** - build a QRecordEntity out of it + ** - change a field (or two) in it + ** - want to pass it into an UpdateAction, and want to see only the fields that + ** you know you changed get passed in to UpdateAction (e.g., PATCH semantics). + ** + ** But also - per the includePrimaryKey param, include the primaryKey in the + ** records (e.g., to tell the Update which records to update). + ** + ** Also, useful for: + ** - construct new entity, calling setters to populate some fields + ** - pass that entity into + *******************************************************************************/ + public QRecord toQRecordOnlyChangedFields(boolean includePrimaryKey) { try { QRecord qRecord = new QRecord(); + String primaryKeyFieldName = ObjectUtils.tryElse(() -> tableReferences.get(getClass().getName()).getPrimaryKeyField(), null); + for(QRecordEntityField qRecordEntityField : getFieldList(this.getClass())) { Serializable thisValue = (Serializable) qRecordEntityField.getGetter().invoke(this); @@ -238,31 +294,16 @@ public abstract class QRecordEntity originalValue = originalRecordValues.get(qRecordEntityField.getFieldName()); } - if(!Objects.equals(thisValue, originalValue)) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this value and the original value don't match - OR - this is the table's primary key field - then put the value in the record. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!Objects.equals(thisValue, originalValue) || (includePrimaryKey && Objects.equals(primaryKeyFieldName, qRecordEntityField.getFieldName()))) { qRecord.setValue(qRecordEntityField.getFieldName(), thisValue); } } - for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass())) - { - @SuppressWarnings("unchecked") - List associatedEntities = (List) qRecordEntityAssociation.getGetter().invoke(this); - String associationName = qRecordEntityAssociation.getAssociationAnnotation().name(); - - if(associatedEntities != null) - { - ///////////////////////////////////////////////////////////////////////////////// - // do this so an empty list in the entity becomes an empty list in the QRecord // - ///////////////////////////////////////////////////////////////////////////////// - qRecord.withAssociatedRecords(associationName, new ArrayList<>()); - } - - for(QRecordEntity associatedEntity : CollectionUtils.nonNullList(associatedEntities)) - { - qRecord.withAssociatedRecord(associationName, associatedEntity.toQRecord()); - } - } + toQRecordProcessAssociations(qRecord, (entity) -> entity.toQRecordOnlyChangedFields(includePrimaryKey)); return (qRecord); } @@ -488,15 +529,16 @@ public abstract class QRecordEntity { // todo - more types!! return (returnType.equals(String.class) - || returnType.equals(Integer.class) - || returnType.equals(int.class) - || returnType.equals(Boolean.class) - || returnType.equals(boolean.class) - || returnType.equals(BigDecimal.class) - || returnType.equals(Instant.class) - || returnType.equals(LocalDate.class) - || returnType.equals(LocalTime.class) - || returnType.equals(byte[].class)); + || returnType.equals(Integer.class) + || returnType.equals(Long.class) + || returnType.equals(int.class) + || returnType.equals(Boolean.class) + || returnType.equals(boolean.class) + || returnType.equals(BigDecimal.class) + || returnType.equals(Instant.class) + || returnType.equals(LocalDate.class) + || returnType.equals(LocalTime.class) + || returnType.equals(byte[].class)); ///////////////////////////////////////////// // note - this list has implications upon: // // - QFieldType.fromClass // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithJoinedRecords.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithJoinedRecords.java new file mode 100644 index 00000000..aa773687 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithJoinedRecords.java @@ -0,0 +1,201 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.data; + + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.BiFunction; + + +/******************************************************************************* + ** Extension on QRecord, intended to be used where you've got records from + ** multiple tables, and you want to combine them into a single "wide" joined + ** record - but to do so without copying or modifying any of the individual + ** records. + ** + ** e.g., given: + ** - Order (id, orderNo, orderDate) (main table) + ** - LineItem (id, sku, quantity) + ** - Extrinsic (id, key, value) + ** + ** If set up in here as: + ** - new QRecordWithJoinedRecords(order) + ** .withJoinedRecordValues(lineItem) + ** .withJoinedRecordValues(extrinsic) + ** + ** Then we'd have the appearance of values in the object like: + ** - id, orderNo, orderDate, lineItem.id, lineItem.sku, lineItem.quantity, extrinsic.id, extrinsic.key, extrinsic.value + ** + ** Which, by the by, is how a query that returns joined records looks, and, is + ** what BackendQueryFilterUtils can use to do filter. + ** + ** This is done without copying or mutating any of the records (which, if you just use + ** QRecord.withJoinedRecordValues, then those values are copied into the main record) + ** - because this object is just storing references to the input records. + ** + ** Note that this implies that, values changed in this record (e.g, calls to setValue) + ** WILL impact the underlying records! + *******************************************************************************/ +public class QRecordWithJoinedRecords extends QRecord +{ + private QRecord mainRecord; + private Map components = new LinkedHashMap<>(); + + + /*************************************************************************** + ** + ***************************************************************************/ + public QRecordWithJoinedRecords(QRecord mainRecord) + { + this.mainRecord = mainRecord; + } + + + + /************************************************************************* + ** + ***************************************************************************/ + @Override + public void addJoinedRecordValues(String joinTableName, QRecord joinedRecord) + { + components.put(joinTableName, joinedRecord); + } + + + + /************************************************************************* + ** + ***************************************************************************/ + public QRecordWithJoinedRecords withJoinedRecordValues(QRecord record, String joinTableName) + { + addJoinedRecordValues(joinTableName, record); + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Serializable getValue(String fieldName) + { + return performFunctionOnRecordBasedOnFieldName(fieldName, ((record, f) -> record.getValue(f))); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void setValue(String fieldName, Object value) + { + performFunctionOnRecordBasedOnFieldName(fieldName, ((record, f) -> + { + record.setValue(f, value); + return (null); + })); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void setValue(String fieldName, Serializable value) + { + performFunctionOnRecordBasedOnFieldName(fieldName, ((record, f) -> + { + record.setValue(f, value); + return (null); + })); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void removeValue(String fieldName) + { + performFunctionOnRecordBasedOnFieldName(fieldName, ((record, f) -> + { + record.removeValue(f); + return (null); + })); + } + + + + /*************************************************************************** + ** avoid having this same block in all the functions that call it... + ** given a fieldName, which may be a joinTable.fieldName, apply the function + ** to the right entity. + ***************************************************************************/ + private Serializable performFunctionOnRecordBasedOnFieldName(String fieldName, BiFunction functionToPerform) + { + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\."); + QRecord component = components.get(parts[0]); + if(component != null) + { + return functionToPerform.apply(component, parts[1]); + } + else + { + return null; + } + } + else + { + return functionToPerform.apply(mainRecord, fieldName); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Map getValues() + { + Map rs = new LinkedHashMap<>(mainRecord.getValues()); + for(Map.Entry componentEntry : components.entrySet()) + { + String joinTableName = componentEntry.getKey(); + QRecord componentRecord = componentEntry.getValue(); + for(Map.Entry entry : componentRecord.getValues().entrySet()) + { + rs.put(joinTableName + "." + entry.getKey(), entry.getValue()); + } + } + return (rs); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java new file mode 100644 index 00000000..04c4e4eb --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java @@ -0,0 +1,47 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import com.kingsrook.qqq.backend.core.logging.QLogger; + + +/******************************************************************************* + ** for use-cases where a metaDataProducer directly adds its objects to the + ** qInstance, then this empty object can be returned. + *******************************************************************************/ +public class EmptyMetaDataProducerOutput implements MetaDataProducerOutput +{ + private static final QLogger LOG = QLogger.getLogger(EmptyMetaDataProducerOutput.class); + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void addSelfToInstance(QInstance instance) + { + ///////////////////////////////// + // noop - this output is empty // + ///////////////////////////////// + LOG.trace("empty meta data producer has nothing to add."); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java index a4dbe375..401ecb0e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java @@ -29,5 +29,40 @@ package com.kingsrook.qqq.backend.core.model.metadata; *******************************************************************************/ public abstract class MetaDataProducer implements MetaDataProducerInterface { + private Class sourceClass; + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + @Override + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + @Override + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public MetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index 1ada252c..ec41e0b6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata; +import java.io.Serializable; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -40,12 +41,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.producers.ChildJoinFromRecordEntityGenericMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.producers.ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; import com.kingsrook.qqq.backend.core.model.metadata.producers.PossibleValueSourceOfEnumGenericMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.producers.PossibleValueSourceOfTableGenericMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.producers.RecordEntityToTableGenericMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.ChildRecordListWidget; import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.ChildTable; import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.QMetaDataProducingEntity; import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.QMetaDataProducingPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.ClassPathUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -82,13 +86,30 @@ public class MetaDataProducerHelper comparatorValuesByType.put(QAppMetaData.class, 23); } + private static MetaDataCustomizerInterface tableMetaDataCustomizer = null; + + + /******************************************************************************* ** Recursively find all classes in the given package, that implement MetaDataProducerInterface - ** run them, and add their output to the given qInstance. + ** run them, and add their output to the given qInstance - using the provided + ** tableMetaDataCustomizer to help with all RecordEntity's that + ** are configured to make tables. ** ** Note - they'll be sorted by the sortOrder they provide. *******************************************************************************/ - public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException + public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName, MetaDataCustomizerInterface tableMetaDataCustomizer) throws QException + { + MetaDataProducerHelper.tableMetaDataCustomizer = tableMetaDataCustomizer; + processAllMetaDataProducersInPackage(instance, packageName); + MetaDataProducerHelper.tableMetaDataCustomizer = null; + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static List> findProducers(String packageName) throws QException { List> classesInPackage; try @@ -116,20 +137,27 @@ public class MetaDataProducerHelper continue; } + ///////////////////////////////////////////////////////////////////// + // handle classes which are themselves MetaDataProducerInterface's // + ///////////////////////////////////////////////////////////////////// if(MetaDataProducerInterface.class.isAssignableFrom(aClass)) { CollectionUtils.addIfNotNull(producers, processMetaDataProducer(aClass)); } + ///////////////////////////////////////////////////////////////////////// + // handle classes that have the @QMetaDataProducingEntity annotation - // + // record entities that should produce meta-data // + ///////////////////////////////////////////////////////////////////////// if(aClass.isAnnotationPresent(QMetaDataProducingEntity.class)) { - QMetaDataProducingEntity qMetaDataProducingEntity = aClass.getAnnotation(QMetaDataProducingEntity.class); - if(qMetaDataProducingEntity.producePossibleValueSource()) - { - producers.addAll(processMetaDataProducingEntity(aClass)); - } + producers.addAll(processMetaDataProducingEntity(aClass)); } + ////////////////////////////////////////////////////////////////// + // handle classes with the @QMetaDataProducingPossibleValueEnum // + // enums that are PVS's // + ////////////////////////////////////////////////////////////////// if(aClass.isAnnotationPresent(QMetaDataProducingPossibleValueEnum.class)) { QMetaDataProducingPossibleValueEnum qMetaDataProducingPossibleValueEnum = aClass.getAnnotation(QMetaDataProducingPossibleValueEnum.class); @@ -164,6 +192,20 @@ public class MetaDataProducerHelper } })); + return (producers); + } + + + /******************************************************************************* + ** Recursively find all classes in the given package, that implement MetaDataProducerInterface + ** run them, and add their output to the given qInstance. + ** + ** Note - they'll be sorted by the sortOrder they provide. + *******************************************************************************/ + public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException + { + List> producers = findProducers(packageName); + /////////////////////////////////////////////////////////////////////////// // execute each one (if enabled), adding their meta data to the instance // /////////////////////////////////////////////////////////////////////////// @@ -197,17 +239,19 @@ public class MetaDataProducerHelper ** ***************************************************************************/ @SuppressWarnings("unchecked") - private static > MetaDataProducerInterface processMetaDataProducingPossibleValueEnum(Class aClass) + private static > MetaDataProducerInterface processMetaDataProducingPossibleValueEnum(Class sourceClass) { String warningPrefix = "Found a class annotated as @" + QMetaDataProducingPossibleValueEnum.class.getSimpleName(); - if(!PossibleValueEnum.class.isAssignableFrom(aClass)) + if(!PossibleValueEnum.class.isAssignableFrom(sourceClass)) { - LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName())); return null; } - PossibleValueEnum[] values = (PossibleValueEnum[]) aClass.getEnumConstants(); - return (new PossibleValueSourceOfEnumGenericMetaDataProducer(aClass.getSimpleName(), (PossibleValueEnum[]) values)); + PossibleValueEnum[] values = (PossibleValueEnum[]) sourceClass.getEnumConstants(); + PossibleValueSourceOfEnumGenericMetaDataProducer producer = new PossibleValueSourceOfEnumGenericMetaDataProducer<>(sourceClass.getSimpleName(), (PossibleValueEnum[]) values); + producer.setSourceClass(sourceClass); + return producer; } @@ -215,41 +259,90 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static List> processMetaDataProducingEntity(Class aClass) throws Exception + private static List> processMetaDataProducingEntity(Class sourceClass) throws Exception { List> rs = new ArrayList<>(); - String warningPrefix = "Found a class annotated as @" + QMetaDataProducingEntity.class.getSimpleName(); - if(!QRecordEntity.class.isAssignableFrom(aClass)) + QMetaDataProducingEntity qMetaDataProducingEntity = sourceClass.getAnnotation(QMetaDataProducingEntity.class); + String warningPrefix = "Found a class annotated as @" + QMetaDataProducingEntity.class.getSimpleName(); + + /////////////////////////////////////////////////////////// + // make sures class is QRecordEntity and cast it as such // + /////////////////////////////////////////////////////////// + if(!QRecordEntity.class.isAssignableFrom(sourceClass)) { - LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName())); return (rs); } - Field tableNameField = aClass.getDeclaredField("TABLE_NAME"); + @SuppressWarnings("unchecked") // safe per the check above. + Class recordEntityClass = (Class) sourceClass; + + //////////////////////////////////////////////// + // get TABLE_NAME static field from the class // + //////////////////////////////////////////////// + Field tableNameField = recordEntityClass.getDeclaredField("TABLE_NAME"); if(!tableNameField.getType().equals(String.class)) { - LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", recordEntityClass.getSimpleName())); return (rs); } String tableNameValue = (String) tableNameField.get(null); - rs.add(new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue)); + + ////////////////////////////////////////// + // add table producer, if so configured // + ////////////////////////////////////////// + if(qMetaDataProducingEntity.produceTableMetaData()) + { + try + { + Class> genericMetaProductionCustomizer = (Class>) qMetaDataProducingEntity.tableMetaDataCustomizer(); + Class> tableMetaDataProductionCustomizer = null; + if(!genericMetaProductionCustomizer.equals(MetaDataCustomizerInterface.NoopMetaDataCustomizer.class)) + { + tableMetaDataProductionCustomizer = (Class>) genericMetaProductionCustomizer; + } + + RecordEntityToTableGenericMetaDataProducer producer = new RecordEntityToTableGenericMetaDataProducer(tableNameValue, recordEntityClass, tableMetaDataProductionCustomizer); + producer.setSourceClass(recordEntityClass); + + if(tableMetaDataCustomizer != null) + { + producer.addRecordEntityTableMetaDataProductionCustomizer(tableMetaDataCustomizer); + } + + rs.add(producer); + } + catch(Exception e) + { + throw new QException("Error processing table meta data producer for entity class: " + recordEntityClass.getName(), e); + } + } + + //////////////////////////////////////// + // add PVS producer, if so configured // + //////////////////////////////////////// + if(qMetaDataProducingEntity.producePossibleValueSource()) + { + PossibleValueSourceOfTableGenericMetaDataProducer producer = new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue); + producer.setSourceClass(recordEntityClass); + rs.add(producer); + } ////////////////////////// // process child tables // ////////////////////////// - QMetaDataProducingEntity qMetaDataProducingEntity = aClass.getAnnotation(QMetaDataProducingEntity.class); for(ChildTable childTable : qMetaDataProducingEntity.childTables()) { Class childEntityClass = childTable.childTableEntityClass(); if(childTable.childJoin().enabled()) { - CollectionUtils.addIfNotNull(rs, processChildJoin(aClass, childTable)); + CollectionUtils.addIfNotNull(rs, processChildJoin(recordEntityClass, childTable)); if(childTable.childRecordListWidget().enabled()) { - CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(aClass, childTable)); + CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(recordEntityClass, childTable)); } } else @@ -259,7 +352,7 @@ public class MetaDataProducerHelper ////////////////////////////////////////////////////////////////////////// // if not doing the join, can't do the child-widget, so warn about that // ////////////////////////////////////////////////////////////////////////// - LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", aClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName())); + LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", recordEntityClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName())); } } } @@ -272,14 +365,16 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processChildRecordListWidget(Class aClass, ChildTable childTable) throws Exception + private static MetaDataProducerInterface processChildRecordListWidget(Class sourceClass, ChildTable childTable) throws Exception { Class childEntityClass = childTable.childTableEntityClass(); - String parentTableName = getTableNameStaticFieldValue(aClass); + String parentTableName = getTableNameStaticFieldValue(sourceClass); String childTableName = getTableNameStaticFieldValue(childEntityClass); ChildRecordListWidget childRecordListWidget = childTable.childRecordListWidget(); - return (new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget)); + ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer producer = new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget); + producer.setSourceClass(sourceClass); + return producer; } @@ -309,20 +404,22 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processChildJoin(Class aClass, ChildTable childTable) throws Exception + private static MetaDataProducerInterface processChildJoin(Class entityClass, ChildTable childTable) throws Exception { Class childEntityClass = childTable.childTableEntityClass(); - String parentTableName = getTableNameStaticFieldValue(aClass); + String parentTableName = getTableNameStaticFieldValue(entityClass); String childTableName = getTableNameStaticFieldValue(childEntityClass); String possibleValueFieldName = findPossibleValueField(childEntityClass, parentTableName); if(!StringUtils.hasContent(possibleValueFieldName)) { - LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + aClass.getSimpleName() + "]"); + LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + entityClass.getSimpleName() + "]"); return (null); } - return (new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName)); + ChildJoinFromRecordEntityGenericMetaDataProducer producer = new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName); + producer.setSourceClass(entityClass); + return producer; } @@ -330,18 +427,20 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processMetaDataProducer(Class aClass) throws Exception + private static MetaDataProducerInterface processMetaDataProducer(Class sourceCClass) throws Exception { - for(Constructor constructor : aClass.getConstructors()) + for(Constructor constructor : sourceCClass.getConstructors()) { if(constructor.getParameterCount() == 0) { Object o = constructor.newInstance(); - return (MetaDataProducerInterface) o; + MetaDataProducerInterface producer = (MetaDataProducerInterface) o; + producer.setSourceClass(sourceCClass); + return producer; } } - LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", aClass.getSimpleName())); + LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", sourceCClass.getSimpleName())); return null; } @@ -361,4 +460,35 @@ public class MetaDataProducerHelper String tableNameValue = (String) tableNameField.get(null); return (tableNameValue); } + + + + /******************************************************************************* + ** Getter for tableMetaDataCustomizer + *******************************************************************************/ + public MetaDataCustomizerInterface getTableMetaDataCustomizer() + { + return (MetaDataProducerHelper.tableMetaDataCustomizer); + } + + + + /******************************************************************************* + ** Setter for tableMetaDataCustomizer + *******************************************************************************/ + public void setTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + MetaDataProducerHelper.tableMetaDataCustomizer = tableMetaDataCustomizer; + } + + + + /******************************************************************************* + ** Fluent setter for tableMetaDataCustomizer + *******************************************************************************/ + public void withTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + MetaDataProducerHelper.tableMetaDataCustomizer = tableMetaDataCustomizer; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java index fa725451..6aef051e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java @@ -73,4 +73,23 @@ public interface MetaDataProducerInterface return (true); } + + /*************************************************************************** + * + ***************************************************************************/ + default void setSourceClass(Class sourceClass) + { + ////////// + // noop // + ////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default Class getSourceClass() + { + return null; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java index d6797a2a..32c446a1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -31,10 +32,12 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; ** Output object for a MetaDataProducer, which contains multiple meta-data ** objects. *******************************************************************************/ -public class MetaDataProducerMultiOutput implements MetaDataProducerOutput +public class MetaDataProducerMultiOutput implements MetaDataProducerOutput, SourceQBitAware { private List contents; + private String sourceQBitName; + /******************************************************************************* @@ -98,4 +101,48 @@ public class MetaDataProducerMultiOutput implements MetaDataProducerOutput return (rs); } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + + ///////////////////////////////////////////// + // propagate the name down to the children // + ///////////////////////////////////////////// + for(MetaDataProducerOutput content : contents) + { + if(content instanceof SourceQBitAware aware) + { + aware.setSourceQBitName(sourceQBitName); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public MetaDataProducerMultiOutput withSourceQBitName(String sourceQBitName) + { + setSourceQBitName(sourceQBitName); + return this; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index 52f6c95f..9bc7ad04 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -26,8 +26,13 @@ import java.util.HashSet; import java.util.Set; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.serialization.QBackendMetaDataDeserializer; import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; +import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -45,21 +50,18 @@ public class QBackendMetaData implements TopLevelMetaDataInterface private Set enabledCapabilities = new HashSet<>(); private Set disabledCapabilities = new HashSet<>(); - private Boolean usesVariants = false; - private String variantOptionsTableIdField; - private String variantOptionsTableNameField; - private String variantOptionsTableTypeField; - private String variantOptionsTableTypeValue; - private String variantOptionsTableUsernameField; - private String variantOptionsTablePasswordField; - private String variantOptionsTableApiKeyField; - private String variantOptionsTableClientIdField; - private String variantOptionsTableClientSecretField; - private String variantOptionsTableName; + private Boolean usesVariants = false; + private BackendVariantsConfig backendVariantsConfig; // todo - at some point, we may want to apply this to secret properties on subclasses? // @JsonFilter("secretsFilter") + @Deprecated(since = "Replaced by filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") + private String variantOptionsTableTypeField; // a field on which to filter the variant-options table, to limit which records in it are available as variants + + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") + private String variantOptionsTableTypeValue; // value for the type-field, to limit which records in it are available as variants; but also, the key in the session.backendVariants map! + /******************************************************************************* @@ -394,22 +396,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface - /******************************************************************************* - ** Getter for variantOptionsTableIdField - *******************************************************************************/ - public String getVariantOptionsTableIdField() - { - return (this.variantOptionsTableIdField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableIdField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key") public void setVariantOptionsTableIdField(String variantOptionsTableIdField) { - this.variantOptionsTableIdField = variantOptionsTableIdField; + ///////////////////////////////////////////////// + // noop as we migrate to backendVariantsConfig // + ///////////////////////////////////////////////// } @@ -417,30 +412,24 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableIdField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key") public QBackendMetaData withVariantOptionsTableIdField(String variantOptionsTableIdField) { - this.variantOptionsTableIdField = variantOptionsTableIdField; + this.setVariantOptionsTableIdField(variantOptionsTableIdField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableNameField - *******************************************************************************/ - public String getVariantOptionsTableNameField() - { - return (this.variantOptionsTableNameField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableNameField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel") public void setVariantOptionsTableNameField(String variantOptionsTableNameField) { - this.variantOptionsTableNameField = variantOptionsTableNameField; + ///////////////////////////////////////////////// + // noop as we migrate to backendVariantsConfig // + ///////////////////////////////////////////////// } @@ -448,30 +437,26 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableNameField *******************************************************************************/ + @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel") public QBackendMetaData withVariantOptionsTableNameField(String variantOptionsTableNameField) { - this.variantOptionsTableNameField = variantOptionsTableNameField; + this.setVariantOptionsTableNameField(variantOptionsTableNameField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableTypeField - *******************************************************************************/ - public String getVariantOptionsTableTypeField() - { - return (this.variantOptionsTableTypeField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableTypeField *******************************************************************************/ + @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField) { this.variantOptionsTableTypeField = variantOptionsTableTypeField; + if(this.variantOptionsTableTypeValue != null) + { + this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue))); + } } @@ -479,30 +464,28 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableTypeField *******************************************************************************/ + @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") public QBackendMetaData withVariantOptionsTableTypeField(String variantOptionsTableTypeField) { - this.variantOptionsTableTypeField = variantOptionsTableTypeField; + this.setVariantOptionsTableTypeField(variantOptionsTableTypeField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableTypeValue - *******************************************************************************/ - public String getVariantOptionsTableTypeValue() - { - return (this.variantOptionsTableTypeValue); - } - - - /******************************************************************************* ** Setter for variantOptionsTableTypeValue *******************************************************************************/ + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue) { + this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeValue); + this.variantOptionsTableTypeValue = variantOptionsTableTypeValue; + if(this.variantOptionsTableTypeField != null) + { + this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue))); + } } @@ -510,30 +493,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableTypeValue *******************************************************************************/ + @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") public QBackendMetaData withVariantOptionsTableTypeValue(String variantOptionsTableTypeValue) { - this.variantOptionsTableTypeValue = variantOptionsTableTypeValue; + this.setVariantOptionsTableTypeValue(variantOptionsTableTypeValue); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableUsernameField - *******************************************************************************/ - public String getVariantOptionsTableUsernameField() - { - return (this.variantOptionsTableUsernameField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableUsernameField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableUsernameField(String variantOptionsTableUsernameField) { - this.variantOptionsTableUsernameField = variantOptionsTableUsernameField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.USERNAME, variantOptionsTableUsernameField); } @@ -541,30 +516,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableUsernameField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableUsernameField(String variantOptionsTableUsernameField) { - this.variantOptionsTableUsernameField = variantOptionsTableUsernameField; + this.setVariantOptionsTableUsernameField(variantOptionsTableUsernameField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTablePasswordField - *******************************************************************************/ - public String getVariantOptionsTablePasswordField() - { - return (this.variantOptionsTablePasswordField); - } - - - /******************************************************************************* ** Setter for variantOptionsTablePasswordField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTablePasswordField(String variantOptionsTablePasswordField) { - this.variantOptionsTablePasswordField = variantOptionsTablePasswordField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.PASSWORD, variantOptionsTablePasswordField); } @@ -572,30 +539,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTablePasswordField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTablePasswordField(String variantOptionsTablePasswordField) { - this.variantOptionsTablePasswordField = variantOptionsTablePasswordField; + this.setVariantOptionsTablePasswordField(variantOptionsTablePasswordField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableApiKeyField - *******************************************************************************/ - public String getVariantOptionsTableApiKeyField() - { - return (this.variantOptionsTableApiKeyField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableApiKeyField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField) { - this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.API_KEY, variantOptionsTableApiKeyField); } @@ -603,30 +562,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableApiKeyField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField) { - this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField; + this.setVariantOptionsTableApiKeyField(variantOptionsTableApiKeyField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableName - *******************************************************************************/ - public String getVariantOptionsTableName() - { - return (this.variantOptionsTableName); - } - - - /******************************************************************************* ** Setter for variantOptionsTableName *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.tableName") public void setVariantOptionsTableName(String variantOptionsTableName) { - this.variantOptionsTableName = variantOptionsTableName; + this.getOrWithNewBackendVariantsConfig().withOptionsTableName(variantOptionsTableName); } @@ -634,9 +585,10 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableName *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.tableName") public QBackendMetaData withVariantOptionsTableName(String variantOptionsTableName) { - this.variantOptionsTableName = variantOptionsTableName; + this.setVariantOptionsTableName(variantOptionsTableName); return (this); } @@ -651,22 +603,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface qInstance.addBackend(this); } - /******************************************************************************* - ** Getter for variantOptionsTableClientIdField - *******************************************************************************/ - public String getVariantOptionsTableClientIdField() - { - return (this.variantOptionsTableClientIdField); - } - /******************************************************************************* ** Setter for variantOptionsTableClientIdField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField) { - this.variantOptionsTableClientIdField = variantOptionsTableClientIdField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_ID, variantOptionsTableClientIdField); } @@ -674,30 +619,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableClientIdField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField) { - this.variantOptionsTableClientIdField = variantOptionsTableClientIdField; + this.setVariantOptionsTableClientIdField(variantOptionsTableClientIdField); return (this); } - /******************************************************************************* - ** Getter for variantOptionsTableClientSecretField - *******************************************************************************/ - public String getVariantOptionsTableClientSecretField() - { - return (this.variantOptionsTableClientSecretField); - } - - - /******************************************************************************* ** Setter for variantOptionsTableClientSecretField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField) { - this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField; + this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_SECRET, variantOptionsTableClientSecretField); } @@ -705,11 +642,55 @@ public class QBackendMetaData implements TopLevelMetaDataInterface /******************************************************************************* ** Fluent setter for variantOptionsTableClientSecretField *******************************************************************************/ + @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap") public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField) { - this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField; + this.setVariantOptionsTableClientSecretField(variantOptionsTableClientSecretField); return (this); } + + /******************************************************************************* + ** Getter for backendVariantsConfig + *******************************************************************************/ + public BackendVariantsConfig getBackendVariantsConfig() + { + return (this.backendVariantsConfig); + } + + + + /******************************************************************************* + ** Setter for backendVariantsConfig + *******************************************************************************/ + public void setBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig) + { + this.backendVariantsConfig = backendVariantsConfig; + } + + + + /******************************************************************************* + ** Fluent setter for backendVariantsConfig + *******************************************************************************/ + public QBackendMetaData withBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig) + { + this.backendVariantsConfig = backendVariantsConfig; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private BackendVariantsConfig getOrWithNewBackendVariantsConfig() + { + if(backendVariantsConfig == null) + { + setBackendVariantsConfig(new BackendVariantsConfig()); + } + return backendVariantsConfig; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index adf62201..cb70eaf4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -56,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; @@ -89,6 +90,7 @@ public class QInstance //////////////////////////////////////////////////////////////////////////////////////////// // Important to use LinkedHashmap here, to preserve the order in which entries are added. // //////////////////////////////////////////////////////////////////////////////////////////// + private Map qBits = new LinkedHashMap<>(); private Map tables = new LinkedHashMap<>(); private Map joins = new LinkedHashMap<>(); private Map possibleValueSources = new LinkedHashMap<>(); @@ -1489,6 +1491,7 @@ public class QInstance } + /******************************************************************************* ** Getter for metaDataFilter *******************************************************************************/ @@ -1519,4 +1522,68 @@ public class QInstance } + + /******************************************************************************* + ** + *******************************************************************************/ + public void addQBit(QBitMetaData qBitMetaData) + { + List missingParts = new ArrayList<>(); + if(!StringUtils.hasContent(qBitMetaData.getGroupId())) + { + missingParts.add("groupId"); + } + if(!StringUtils.hasContent(qBitMetaData.getArtifactId())) + { + missingParts.add("artifactId"); + } + if(!StringUtils.hasContent(qBitMetaData.getVersion())) + { + missingParts.add("version"); + + } + if(!missingParts.isEmpty()) + { + throw (new IllegalArgumentException("Attempted to add a qBit without a " + StringUtils.joinWithCommasAndAnd(missingParts))); + } + + String name = qBitMetaData.getName(); + if(this.qBits.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second qBit with name (formed from 'groupId:artifactId:version[:namespace]'): " + name)); + } + this.qBits.put(name, qBitMetaData); + } + + + + /******************************************************************************* + ** Getter for qBits + *******************************************************************************/ + public Map getQBits() + { + return (this.qBits); + } + + + + /******************************************************************************* + ** Setter for qBits + *******************************************************************************/ + public void setQBits(Map qBits) + { + this.qBits = qBits; + } + + + + /******************************************************************************* + ** Fluent setter for qBits + *******************************************************************************/ + public QInstance withQBits(Map qBits) + { + this.qBits = qBits; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java index e80e2d68..185b0035 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java @@ -30,4 +30,5 @@ public enum AuditLevel NONE, RECORD, FIELD + // idea: only audit changes to fields, e.g., on edit. though, is that a different dimension than this? } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java index e7793b03..d5c63d04 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; @@ -69,6 +70,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface protected Map defaultValues = new LinkedHashMap<>(); + protected QInstanceValidatorPluginInterface validatorPlugin; /******************************************************************************* @@ -764,4 +766,35 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot); } + + /******************************************************************************* + ** Getter for validatorPlugin + *******************************************************************************/ + public QInstanceValidatorPluginInterface getValidatorPlugin() + { + return (this.validatorPlugin); + } + + + + /******************************************************************************* + ** Setter for validatorPlugin + *******************************************************************************/ + public void setValidatorPlugin(QInstanceValidatorPluginInterface validatorPlugin) + { + this.validatorPlugin = validatorPlugin; + } + + + + /******************************************************************************* + ** Fluent setter for validatorPlugin + *******************************************************************************/ + public QWidgetMetaData withValidatorPlugin(QInstanceValidatorPluginInterface validatorPlugin) + { + this.validatorPlugin = validatorPlugin; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java index ed8af4e3..e9c9b54a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Set; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; @@ -277,5 +278,13 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, T qInstance.addWidget(this); } + + /*************************************************************************** + ** let the widget include an instance validator plugin + ***************************************************************************/ + default QInstanceValidatorPluginInterface getValidatorPlugin() + { + return (null); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java index 7e3115e5..4cf6e42b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java @@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; import java.io.Serializable; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Objects; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; @@ -41,20 +45,22 @@ public enum AdornmentType RENDER_HTML, REVEAL, FILE_DOWNLOAD, + FILE_UPLOAD, + TOOLTIP, ERROR; ////////////////////////////////////////////////////////////////////////// // keep these values in sync with AdornmentType.ts in qqq-frontend-core // ////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** *******************************************************************************/ public interface LinkValues { - String TARGET = "target"; - String TO_RECORD_FROM_TABLE = "toRecordFromTable"; + String TARGET = "target"; + String TO_RECORD_FROM_TABLE = "toRecordFromTable"; + String TO_RECORD_FROM_TABLE_DYNAMIC = "toRecordFromTableDynamic"; } @@ -71,6 +77,8 @@ public enum AdornmentType String SUPPLEMENTAL_PROCESS_NAME = "supplementalProcessName"; String SUPPLEMENTAL_CODE_REFERENCE = "supplementalCodeReference"; + String DOWNLOAD_URL_DYNAMIC = "downloadUrlDynamic"; + //////////////////////////////////////////////////// // use these two together, as in: // // FILE_NAME_FORMAT = "Order %s Packing Slip.pdf" // @@ -78,6 +86,17 @@ public enum AdornmentType //////////////////////////////////////////////////// String FILE_NAME_FORMAT = "fileNameFormat"; String FILE_NAME_FORMAT_FIELDS = "fileNameFormatFields"; + + /*************************************************************************** + ** + ***************************************************************************/ + static String makeFieldDownloadUrl(String tableName, Serializable primaryKey, String fieldName, String fileName) + { + return ("/data/" + tableName + "/" + + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8).replace("+", "%20") + "/" + + fieldName + "/" + + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8).replace("+", "%20")); + } } @@ -167,4 +186,76 @@ public enum AdornmentType } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class FileUploadAdornment + { + public static String FORMAT = "format"; + public static String WIDTH = "width"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static FieldAdornment newFieldAdornment() + { + return (new FieldAdornment(AdornmentType.FILE_UPLOAD)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair formatDragAndDrop() + { + return (Pair.of(FORMAT, "dragAndDrop")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair formatButton() + { + return (Pair.of(FORMAT, "button")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair widthFull() + { + return (Pair.of(WIDTH, "full")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair widthHalf() + { + return (Pair.of(WIDTH, "half")); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public interface TooltipValues + { + String STATIC_TEXT = "staticText"; + String TOOLTIP_DYNAMIC = "tooltipDynamic"; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java index b8aa82c5..74cc9db7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java @@ -177,7 +177,7 @@ public class FieldAdornment ** Fluent setter for values ** *******************************************************************************/ - public FieldAdornment withValue(Pair value) + public FieldAdornment withValue(Pair value) { return (withValue(value.getA(), value.getB())); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAndJoinTable.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAndJoinTable.java new file mode 100644 index 00000000..9b8b348d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAndJoinTable.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.fields; + + +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Wrapper (record) that holds a QFieldMetaData and a QTableMetaData - + ** + ** With a factory method (`get()`) to go from the use-case of, a String that's + ** "joinTable.fieldName" or "fieldName" to the pair. + ** + ** Note that the "joinTable" member here - could be the "mainTable" passed in + ** to that `get()` method. + ** + *******************************************************************************/ +public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) +{ + + /*************************************************************************** + ** given a table, and a field-name string (which should either be the name + ** of a field on that table, or another tableName + "." + fieldName (from + ** that table) - get back the pair of table & field metaData that the + ** input string is talking about. + ***************************************************************************/ + public static FieldAndJoinTable get(QTableMetaData mainTable, String fieldName) throws QException + { + if(fieldName.indexOf('.') > -1) + { + String joinTableName = fieldName.replaceAll("\\..*", ""); + String joinFieldName = fieldName.replaceAll(".*\\.", ""); + + QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName); + if(joinTable == null) + { + throw (new QException("Unrecognized join table name: " + joinTableName)); + } + + return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable); + } + else + { + return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public String getLabel(QTableMetaData mainTable) + { + if(mainTable.getName().equals(joinTable.getName())) + { + return (field.getLabel()); + } + else + { + return (joinTable.getLabel() + ": " + field.getLabel()); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java index 5ea1cfa3..9fbdecdf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldDisplayBehavior.java @@ -22,11 +22,45 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + /******************************************************************************* ** Interface to mark a field behavior as one to be used during generating ** display values. *******************************************************************************/ public interface FieldDisplayBehavior> extends FieldBehavior { + NoopFieldDisplayBehavior NOOP = new NoopFieldDisplayBehavior(); + /*************************************************************************** + ** + ***************************************************************************/ + @Override + @SuppressWarnings("unchecked") + default T getDefault() + { + return (T) NOOP; + } + + + /*************************************************************************** + ** a default implementation for this behavior type, which does nothing. + ***************************************************************************/ + class NoopFieldDisplayBehavior implements FieldDisplayBehavior + { + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FilterJsonFieldDisplayValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FilterJsonFieldDisplayValueFormatter.java new file mode 100644 index 00000000..00b7475e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FilterJsonFieldDisplayValueFormatter.java @@ -0,0 +1,73 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.fields; + + +import java.util.List; +import java.util.function.Consumer; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Display value formatter for fields which store a QQueryFilter as JSON. + *******************************************************************************/ +public class FilterJsonFieldDisplayValueFormatter implements FieldDisplayBehavior +{ + private static Consumer jsonMapperCustomizer = om -> om.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + String queryFilterJson = record.getValueString(field.getName()); + if(StringUtils.hasContent(queryFilterJson)) + { + try + { + QQueryFilter qQueryFilter = JsonUtils.toObject(queryFilterJson, QQueryFilter.class, jsonMapperCustomizer); + int criteriaCount = CollectionUtils.nonNullList(qQueryFilter.getCriteria()).size(); + record.setDisplayValue(field.getName(), criteriaCount + " Filter" + StringUtils.plural(criteriaCount)); + } + catch(Exception e) + { + record.setDisplayValue(field.getName(), "Invalid Filter..."); + } + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index 0de975e0..12eb5454 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -82,6 +82,7 @@ public class QFieldMetaData implements Cloneable, QMetaDataObject private QQueryFilter possibleValueSourceFilter; private QPossibleValueSource inlinePossibleValueSource; + private Integer gridColumns; private Integer maxLength; private Set> behaviors; @@ -199,6 +200,7 @@ public class QFieldMetaData implements Cloneable, QMetaDataObject setIsRequired(fieldAnnotation.isRequired()); setIsEditable(fieldAnnotation.isEditable()); setIsHidden(fieldAnnotation.isHidden()); + setGridColumns(fieldAnnotation.gridColumns()); if(StringUtils.hasContent(fieldAnnotation.label())) { @@ -1063,6 +1065,7 @@ public class QFieldMetaData implements Cloneable, QMetaDataObject } + /******************************************************************************* ** Getter for inlinePossibleValueSource *******************************************************************************/ @@ -1093,4 +1096,34 @@ public class QFieldMetaData implements Cloneable, QMetaDataObject } + + /******************************************************************************* + ** Getter for gridColumns + *******************************************************************************/ + public Integer getGridColumns() + { + return (this.gridColumns); + } + + + + /******************************************************************************* + ** Setter for gridColumns + *******************************************************************************/ + public void setGridColumns(Integer gridColumns) + { + this.gridColumns = gridColumns; + } + + + + /******************************************************************************* + ** Fluent setter for gridColumns + *******************************************************************************/ + public QFieldMetaData withGridColumns(Integer gridColumns) + { + this.gridColumns = gridColumns; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index 69d80c1d..865bdc4d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -27,6 +27,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -100,6 +101,16 @@ public enum QFieldType + /*************************************************************************** + ** + ***************************************************************************/ + public String getMixedCaseLabel() + { + return StringUtils.allCapsToMixedCase(name()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueRangeBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueRangeBehavior.java new file mode 100644 index 00000000..aed2317a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueRangeBehavior.java @@ -0,0 +1,477 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.fields; + + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + + +/******************************************************************************* + ** Validate the min & max value for numeric fields. + ** + ** For each min & max, there are 4 possible settings: + ** - value - the number that is compared. + ** - allowEqualTo - defaults to true. controls if < (>) or ≤ (≥) + ** - behavior - defaults to ERROR. optionally can be "CLIP" instead. + ** - clipAmount - if clipping, and not allowing equalTo, how much off the limit + ** value should be added or subtracted. Defaults to 1. + ** + ** Convenient `withMin()` and `withMax()` methods exist for setting all 4 + ** properties for each of min or max. Else, fluent-setters are recommended. + *******************************************************************************/ +public class ValueRangeBehavior implements FieldBehavior +{ + /*************************************************************************** + ** + ***************************************************************************/ + public enum Behavior + { + ERROR, + CLIP + } + + + + private Number minValue; + private boolean minAllowEqualTo = true; + private Behavior minBehavior = Behavior.ERROR; + private BigDecimal minClipAmount = BigDecimal.ONE; + + private Number maxValue; + private boolean maxAllowEqualTo = true; + private Behavior maxBehavior = Behavior.ERROR; + private BigDecimal maxClipAmount = BigDecimal.ONE; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ValueRangeBehavior() + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public ValueRangeBehavior getDefault() + { + return null; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + BigDecimal minLimitBigDecimal = minValue == null ? null : new BigDecimal(minValue.toString()); + String minLimitString = minValue == null ? null : minValue.toString(); + + BigDecimal maxLimitBigDecimal = maxValue == null ? null : new BigDecimal(maxValue.toString()); + String maxLimitString = maxValue == null ? null : maxValue.toString(); + + for(QRecord record : recordList) + { + BigDecimal recordValue = record.getValueBigDecimal(field.getName()); + if(recordValue != null) + { + if(minLimitBigDecimal != null) + { + int compare = recordValue.compareTo(minLimitBigDecimal); + if(compare < 0 || (compare == 0 && !minAllowEqualTo)) + { + if(this.minBehavior == Behavior.ERROR) + { + String operator = minAllowEqualTo ? "" : "greater than "; + record.addError(new BadInputStatusMessage("The value for " + field.getLabel() + " is too small (minimum allowed value is " + operator + minLimitString + ")")); + } + else if(this.minBehavior == Behavior.CLIP) + { + if(minAllowEqualTo) + { + record.setValue(field.getName(), minLimitBigDecimal); + } + else + { + record.setValue(field.getName(), minLimitBigDecimal.add(minClipAmount)); + } + } + } + } + + if(maxLimitBigDecimal != null) + { + int compare = recordValue.compareTo(maxLimitBigDecimal); + if(compare > 0 || (compare == 0 && !maxAllowEqualTo)) + { + if(this.maxBehavior == Behavior.ERROR) + { + String operator = maxAllowEqualTo ? "" : "less than "; + record.addError(new BadInputStatusMessage("The value for " + field.getLabel() + " is too large (maximum allowed value is " + operator + maxLimitString + ")")); + } + else if(this.maxBehavior == Behavior.CLIP) + { + if(maxAllowEqualTo) + { + record.setValue(field.getName(), maxLimitBigDecimal); + } + else + { + record.setValue(field.getName(), maxLimitBigDecimal.subtract(maxClipAmount)); + } + } + } + } + + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean allowMultipleBehaviorsOfThisType() + { + return (false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List validateBehaviorConfiguration(QTableMetaData tableMetaData, QFieldMetaData fieldMetaData) + { + List errors = new ArrayList<>(); + + if(minValue == null && maxValue == null) + { + errors.add("Either minValue or maxValue (or both) must be set."); + } + + if(minValue != null && maxValue != null && new BigDecimal(minValue.toString()).compareTo(new BigDecimal(maxValue.toString())) > 0) + { + errors.add("minValue must be >= maxValue."); + } + + if(fieldMetaData != null && fieldMetaData.getType() != null && !fieldMetaData.getType().isNumeric()) + { + errors.add("can only be applied to a numeric type field."); + } + + return (errors); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public ValueRangeBehavior withMin(Number value, boolean allowEqualTo, Behavior behavior, BigDecimal clipAmount) + { + setMinValue(value); + setMinAllowEqualTo(allowEqualTo); + setMinBehavior(behavior); + setMinClipAmount(clipAmount); + return (this); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public ValueRangeBehavior withMax(Number value, boolean allowEqualTo, Behavior behavior, BigDecimal clipAmount) + { + setMaxValue(value); + setMaxAllowEqualTo(allowEqualTo); + setMaxBehavior(behavior); + setMaxClipAmount(clipAmount); + return (this); + } + + + + /******************************************************************************* + ** Getter for minValue + *******************************************************************************/ + public Number getMinValue() + { + return (this.minValue); + } + + + + /******************************************************************************* + ** Setter for minValue + *******************************************************************************/ + public void setMinValue(Number minValue) + { + this.minValue = minValue; + } + + + + /******************************************************************************* + ** Fluent setter for minValue + *******************************************************************************/ + public ValueRangeBehavior withMinValue(Number minValue) + { + this.minValue = minValue; + return (this); + } + + + + /******************************************************************************* + ** Getter for maxValue + *******************************************************************************/ + public Number getMaxValue() + { + return (this.maxValue); + } + + + + /******************************************************************************* + ** Setter for maxValue + *******************************************************************************/ + public void setMaxValue(Number maxValue) + { + this.maxValue = maxValue; + } + + + + /******************************************************************************* + ** Fluent setter for maxValue + *******************************************************************************/ + public ValueRangeBehavior withMaxValue(Number maxValue) + { + this.maxValue = maxValue; + return (this); + } + + + + /******************************************************************************* + ** Getter for minAllowEqualTo + *******************************************************************************/ + public boolean getMinAllowEqualTo() + { + return (this.minAllowEqualTo); + } + + + + /******************************************************************************* + ** Setter for minAllowEqualTo + *******************************************************************************/ + public void setMinAllowEqualTo(boolean minAllowEqualTo) + { + this.minAllowEqualTo = minAllowEqualTo; + } + + + + /******************************************************************************* + ** Fluent setter for minAllowEqualTo + *******************************************************************************/ + public ValueRangeBehavior withMinAllowEqualTo(boolean minAllowEqualTo) + { + this.minAllowEqualTo = minAllowEqualTo; + return (this); + } + + + + /******************************************************************************* + ** Getter for maxAllowEqualTo + *******************************************************************************/ + public boolean getMaxAllowEqualTo() + { + return (this.maxAllowEqualTo); + } + + + + /******************************************************************************* + ** Setter for maxAllowEqualTo + *******************************************************************************/ + public void setMaxAllowEqualTo(boolean maxAllowEqualTo) + { + this.maxAllowEqualTo = maxAllowEqualTo; + } + + + + /******************************************************************************* + ** Fluent setter for maxAllowEqualTo + *******************************************************************************/ + public ValueRangeBehavior withMaxAllowEqualTo(boolean maxAllowEqualTo) + { + this.maxAllowEqualTo = maxAllowEqualTo; + return (this); + } + + + + /******************************************************************************* + ** Getter for minBehavior + *******************************************************************************/ + public Behavior getMinBehavior() + { + return (this.minBehavior); + } + + + + /******************************************************************************* + ** Setter for minBehavior + *******************************************************************************/ + public void setMinBehavior(Behavior minBehavior) + { + this.minBehavior = minBehavior; + } + + + + /******************************************************************************* + ** Fluent setter for minBehavior + *******************************************************************************/ + public ValueRangeBehavior withMinBehavior(Behavior minBehavior) + { + this.minBehavior = minBehavior; + return (this); + } + + + + /******************************************************************************* + ** Getter for maxBehavior + *******************************************************************************/ + public Behavior getMaxBehavior() + { + return (this.maxBehavior); + } + + + + /******************************************************************************* + ** Setter for maxBehavior + *******************************************************************************/ + public void setMaxBehavior(Behavior maxBehavior) + { + this.maxBehavior = maxBehavior; + } + + + + /******************************************************************************* + ** Fluent setter for maxBehavior + *******************************************************************************/ + public ValueRangeBehavior withMaxBehavior(Behavior maxBehavior) + { + this.maxBehavior = maxBehavior; + return (this); + } + + + + /******************************************************************************* + ** Getter for minClipAmount + *******************************************************************************/ + public BigDecimal getMinClipAmount() + { + return (this.minClipAmount); + } + + + + /******************************************************************************* + ** Setter for minClipAmount + *******************************************************************************/ + public void setMinClipAmount(BigDecimal minClipAmount) + { + this.minClipAmount = minClipAmount; + } + + + + /******************************************************************************* + ** Fluent setter for minClipAmount + *******************************************************************************/ + public ValueRangeBehavior withMinClipAmount(BigDecimal minClipAmount) + { + this.minClipAmount = minClipAmount; + return (this); + } + + + + /******************************************************************************* + ** Getter for maxClipAmount + *******************************************************************************/ + public BigDecimal getMaxClipAmount() + { + return (this.maxClipAmount); + } + + + + /******************************************************************************* + ** Setter for maxClipAmount + *******************************************************************************/ + public void setMaxClipAmount(BigDecimal maxClipAmount) + { + this.maxClipAmount = maxClipAmount; + } + + + + /******************************************************************************* + ** Fluent setter for maxClipAmount + *******************************************************************************/ + public ValueRangeBehavior withMaxClipAmount(BigDecimal maxClipAmount) + { + this.maxClipAmount = maxClipAmount; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java index 38dfdc54..c6c6014f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java @@ -43,7 +43,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; * *******************************************************************************/ @JsonInclude(Include.NON_NULL) -public class QFrontendFieldMetaData +public class QFrontendFieldMetaData implements Serializable { private String name; private String label; @@ -51,6 +51,7 @@ public class QFrontendFieldMetaData private boolean isRequired; private boolean isEditable; private boolean isHeavy; + private Integer gridColumns; private String possibleValueSourceName; private String displayFormat; private Serializable defaultValue; @@ -66,7 +67,6 @@ public class QFrontendFieldMetaData ////////////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** Constructor *******************************************************************************/ @@ -78,6 +78,7 @@ public class QFrontendFieldMetaData this.isRequired = fieldMetaData.getIsRequired(); this.isEditable = fieldMetaData.getIsEditable(); this.isHeavy = fieldMetaData.getIsHeavy(); + this.gridColumns = fieldMetaData.getGridColumns(); this.possibleValueSourceName = fieldMetaData.getPossibleValueSourceName(); this.displayFormat = fieldMetaData.getDisplayFormat(); this.adornments = fieldMetaData.getAdornments(); @@ -166,6 +167,17 @@ public class QFrontendFieldMetaData + /******************************************************************************* + ** Getter for gridColumns + ** + *******************************************************************************/ + public Integer getGridColumns() + { + return gridColumns; + } + + + /******************************************************************************* ** Getter for displayFormat ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index 6e98fdec..01b7832e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -86,7 +86,6 @@ public class QFrontendTableMetaData ////////////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** *******************************************************************************/ @@ -170,7 +169,7 @@ public class QFrontendTableMetaData if(backend != null && backend.getUsesVariants()) { usesVariants = true; - variantTableLabel = QContext.getQInstance().getTable(backend.getVariantOptionsTableName()).getLabel(); + variantTableLabel = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()).getLabel(); } this.helpContents = tableMetaData.getHelpContent(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java index 53e13f54..f3b3e2de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java @@ -216,11 +216,16 @@ public class SendSESAction { LOG.warn("More than one FROM value was found, will send using the first one found [" + partyList.get(0).getAddress() + "]."); } + Party fromParty = partyList.get(0); + if(fromParty.getAddress() == null) + { + throw (new QException("Cannot send SES message because a FROM address was not provided.")); + } ///////////////////////////// // return the from address // ///////////////////////////// - return (partyList.get(0).getAddress()); + return (getFullEmailAddress(fromParty)); } @@ -267,15 +272,15 @@ public class SendSESAction { if(EmailPartyRole.CC.equals(party.getRole())) { - ccList.add(party.getAddress()); + ccList.add(getFullEmailAddress(party)); } else if(EmailPartyRole.BCC.equals(party.getRole())) { - bccList.add(party.getAddress()); + bccList.add(getFullEmailAddress(party)); } else if(party.getRole() == null || PartyRole.Default.DEFAULT.equals(party.getRole()) || EmailPartyRole.TO.equals(party.getRole())) { - toList.add(party.getAddress()); + toList.add(getFullEmailAddress(party)); } else { @@ -332,4 +337,22 @@ public class SendSESAction return amazonSES; } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullEmailAddress(Party party) + { + if(party.getLabel() != null) + { + return (party.getLabel() + " <" + party.getAddress() + ">"); + } + + ///////////////////////////// + // return the from address // + ///////////////////////////// + return (party.getAddress()); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java index fc61b2df..97585230 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java @@ -22,11 +22,14 @@ package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues; +import java.io.Serializable; + + /******************************************************************************* ** Interface to be implemented by enums which can be used as a PossibleValueSource. ** *******************************************************************************/ -public interface PossibleValueEnum +public interface PossibleValueEnum { /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java index 40a9dde2..7ff385e8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues; +import java.io.Serializable; import java.util.Objects; @@ -30,7 +31,7 @@ import java.util.Objects; ** ** Type parameter `T` is the type of the id (often Integer, maybe String) *******************************************************************************/ -public class QPossibleValue +public class QPossibleValue { private final T id; private final String label; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java index 84a24914..4cb4636a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues; +import java.io.Serializable; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -31,6 +32,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import net.sf.saxon.trans.SaxonErrorCode; /******************************************************************************* @@ -45,6 +48,8 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface private String label; private QPossibleValueSourceType type; + private QFieldType idType; + private String valueFormat = PVSValueFormatAndFields.LABEL_ONLY.getFormat(); private List valueFields = PVSValueFormatAndFields.LABEL_ONLY.getFields(); private String valueFormatIfNotFound = null; @@ -100,7 +105,7 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface ** Create a new possible value source, for an enum, with default settings. ** e.g., type=ENUM; name from param values from the param; LABEL_ONLY format *******************************************************************************/ - public static > QPossibleValueSource newForEnum(String name, T[] values) + public static > QPossibleValueSource newForEnum(String name, T[] values) { return new QPossibleValueSource() .withName(name) @@ -556,7 +561,7 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface ** myPossibleValueSource.withValuesFromEnum(MyEnum.values())); ** *******************************************************************************/ - public > QPossibleValueSource withValuesFromEnum(T[] values) + public > QPossibleValueSource withValuesFromEnum(T[] values) { Set usedIds = new HashSet<>(); List duplicatedIds = new ArrayList<>(); @@ -679,4 +684,35 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface return (this); } + + /******************************************************************************* + ** Getter for idType + *******************************************************************************/ + public QFieldType getIdType() + { + return (this.idType); + } + + + + /******************************************************************************* + ** Setter for idType + *******************************************************************************/ + public void setIdType(QFieldType idType) + { + this.idType = idType; + } + + + + /******************************************************************************* + ** Fluent setter for idType + *******************************************************************************/ + public QPossibleValueSource withIdType(QFieldType idType) + { + this.idType = idType; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java index d4bd7ff7..c15eba2a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java @@ -29,6 +29,9 @@ public enum QComponentType { HELP_TEXT, BULK_EDIT_FORM, + BULK_LOAD_FILE_MAPPING_FORM, + BULK_LOAD_VALUE_MAPPING_FORM, + BULK_LOAD_PROFILE_FORM, VALIDATION_REVIEW_SCREEN, EDIT_FORM, VIEW_FORM, diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java index 0c07043e..514e4fae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java @@ -48,6 +48,7 @@ public class QFrontendStepMetaData extends QStepMetaData private Map formFieldMap; private String format; + private String backStepName; private List helpContents; @@ -436,4 +437,35 @@ public class QFrontendStepMetaData extends QStepMetaData } + + /******************************************************************************* + ** Getter for backStepName + *******************************************************************************/ + public String getBackStepName() + { + return (this.backStepName); + } + + + + /******************************************************************************* + ** Setter for backStepName + *******************************************************************************/ + public void setBackStepName(String backStepName) + { + this.backStepName = backStepName; + } + + + + /******************************************************************************* + ** Fluent setter for backStepName + *******************************************************************************/ + public QFrontendStepMetaData withBackStepName(String backStepName) + { + this.backStepName = backStepName; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index d4867166..baaa16ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -31,11 +31,13 @@ import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -45,11 +47,14 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; ** Meta-Data to define a process in a QQQ instance. ** *******************************************************************************/ -public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface +public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface, SourceQBitAware { - private String name; - private String label; - private String tableName; + private String name; + private String label; + private String tableName; + + private String sourceQBitName; + private boolean isHidden = false; private BasepullConfiguration basepullConfiguration; private QPermissionRules permissionRules; @@ -70,6 +75,8 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi private VariantRunStrategy variantRunStrategy; private String variantBackend; + private QCodeReference processTracerCodeReference; + private Map supplementalMetaData; @@ -877,4 +884,69 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi return (this); } + + + /******************************************************************************* + ** Getter for processTracerCodeReference + *******************************************************************************/ + public QCodeReference getProcessTracerCodeReference() + { + return (this.processTracerCodeReference); + } + + + + /******************************************************************************* + ** Setter for processTracerCodeReference + *******************************************************************************/ + public void setProcessTracerCodeReference(QCodeReference processTracerCodeReference) + { + this.processTracerCodeReference = processTracerCodeReference; + } + + + + /******************************************************************************* + ** Fluent setter for processTracerCodeReference + *******************************************************************************/ + public QProcessMetaData withProcessTracerCodeReference(QCodeReference processTracerCodeReference) + { + this.processTracerCodeReference = processTracerCodeReference; + return (this); + } + + + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + @Override + public QProcessMetaData withSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java index 76c892cd..c1bc51cf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java @@ -62,6 +62,7 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat private String parentTableName; // e.g., order private String foreignKeyFieldName; // e.g., orderId + private Class sourceClass; /*************************************************************************** @@ -102,4 +103,37 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat return (join); } + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public ChildJoinFromRecordEntityGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java index cb8d451f..6dac1a10 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java @@ -38,14 +38,14 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; ** produce a QJoinMetaData, based on a QRecordEntity and a ChildTable sub-annotation. ** ** e.g., Orders & LineItems - on the Order entity - ** + @QMetaDataProducingEntity( childTables = { @ChildTable( - childTableEntityClass = LineItem.class, - childJoin = @ChildJoin(enabled = true), - childRecordListWidget = @ChildRecordListWidget(enabled = true, label = "Order Lines")) + childTableEntityClass = LineItem.class, + childJoin = @ChildJoin(enabled = true), + childRecordListWidget = @ChildRecordListWidget(enabled = true, label = "Order Lines")) }) public class Order extends QRecordEntity - ** + ** *******************************************************************************/ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implements MetaDataProducerInterface @@ -53,18 +53,29 @@ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implem private String childTableName; // e.g., lineItem private String parentTableName; // e.g., order + private MetaDataCustomizerInterface widgetMetaDataProductionCustomizer = null; + private ChildRecordListWidget childRecordListWidget; + private Class sourceClass; + /*************************************************************************** ** ***************************************************************************/ - public ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(String childTableName, String parentTableName, ChildRecordListWidget childRecordListWidget) + public ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(String childTableName, String parentTableName, ChildRecordListWidget childRecordListWidget) throws Exception { this.childTableName = childTableName; this.parentTableName = parentTableName; this.childRecordListWidget = childRecordListWidget; + + Class> genericMetaProductionCustomizer = (Class>) childRecordListWidget.widgetMetaDataCustomizer(); + if(!genericMetaProductionCustomizer.equals(MetaDataCustomizerInterface.NoopMetaDataCustomizer.class)) + { + Class> widgetMetaProductionCustomizerClass = (Class>) genericMetaProductionCustomizer; + this.widgetMetaDataProductionCustomizer = widgetMetaProductionCustomizerClass.getConstructor().newInstance(); + } } @@ -94,7 +105,44 @@ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implem widget.withDefaultValue("maxRows", childRecordListWidget.maxRows()); } + if(this.widgetMetaDataProductionCustomizer != null) + { + widget = this.widgetMetaDataProductionCustomizer.customizeMetaData(qInstance, widget); + } + return (widget); } + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/MetaDataCustomizerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/MetaDataCustomizerInterface.java new file mode 100644 index 00000000..248fd79e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/MetaDataCustomizerInterface.java @@ -0,0 +1,58 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.producers; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; + + +/******************************************************************************* + ** Interface to be implemented by classes that are designed to help customize + ** meta-data objects as they're being produced, e.g., such as a table produced + ** via the QMetaDataProducingEntity, or maybe tables loaded by a qbit?? + *******************************************************************************/ +public interface MetaDataCustomizerInterface +{ + /*************************************************************************** + ** + ***************************************************************************/ + T customizeMetaData(QInstance qInstance, T metaData) throws QException; + + + /*************************************************************************** + ** noop version of this interface - used as default value in annotation + ** + ***************************************************************************/ + class NoopMetaDataCustomizer implements MetaDataCustomizerInterface + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public T customizeMetaData(QInstance qInstance, T metaData) throws QException + { + return (metaData); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java index 52fbdffa..f43818b5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.producers; +import java.io.Serializable; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; @@ -34,11 +35,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal ** based on a PossibleValueEnum ** ***************************************************************************/ -public class PossibleValueSourceOfEnumGenericMetaDataProducer> implements MetaDataProducerInterface +public class PossibleValueSourceOfEnumGenericMetaDataProducer> implements MetaDataProducerInterface { private final String name; private final PossibleValueEnum[] values; + private Class sourceClass; + + + /******************************************************************************* @@ -61,4 +66,37 @@ public class PossibleValueSourceOfEnumGenericMetaDataProducer getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public PossibleValueSourceOfEnumGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java index c1656f35..7d194f64 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java @@ -37,6 +37,7 @@ public class PossibleValueSourceOfTableGenericMetaDataProducer implements MetaDa { private final String tableName; + private Class sourceClass; /******************************************************************************* @@ -58,4 +59,38 @@ public class PossibleValueSourceOfTableGenericMetaDataProducer implements MetaDa { return (QPossibleValueSource.newForTable(tableName)); } + + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public PossibleValueSourceOfTableGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java new file mode 100644 index 00000000..20cf5be1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java @@ -0,0 +1,191 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.producers; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/*************************************************************************** + ** Generic meta-data-producer, which should be instantiated (e.g., by + ** MetaDataProducerHelper), to produce a QPossibleValueSource meta-data + ** based on a QRecordEntity class (which has corresponding QTableMetaData). + ** + ***************************************************************************/ +public class RecordEntityToTableGenericMetaDataProducer implements MetaDataProducerInterface +{ + private final String tableName; + private final Class entityClass; + + private final List> metaDataCustomizers = new ArrayList<>(); + + private static MetaDataCustomizerInterface defaultMetaDataCustomizer = null; + + private Class sourceClass; + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RecordEntityToTableGenericMetaDataProducer(String tableName, Class entityClass, Class> metaDataProductionCustomizerClass) throws QException + { + this.tableName = tableName; + this.entityClass = entityClass; + + if(metaDataProductionCustomizerClass != null) + { + metaDataCustomizers.add(getMetaDataProductionCustomizer(metaDataProductionCustomizerClass)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + QTableMetaData qTableMetaData = new QTableMetaData(); + qTableMetaData.setName(tableName); + qTableMetaData.setRecordLabelFormat("%s"); + qTableMetaData.withFieldsFromEntity(entityClass); + + //////////////////////////////////////////////////////////////////// + // use the productionCustomizers to fill in more of the meta data // + //////////////////////////////////////////////////////////////////// + for(MetaDataCustomizerInterface metaDataMetaDataCustomizer : metaDataCustomizers) + { + qTableMetaData = metaDataMetaDataCustomizer.customizeMetaData(qInstance, qTableMetaData); + } + + /////////////////////////////////////////////////////////////////////////////////// + // now if there's a default customizer, call it too - for generic, common things // + // you might want on all of your tables, or defaults if not set otherwise // + /////////////////////////////////////////////////////////////////////////////////// + if(defaultMetaDataCustomizer != null) + { + qTableMetaData = defaultMetaDataCustomizer.customizeMetaData(qInstance, qTableMetaData); + } + + ///////////////////////////////////////////////////////////////////////// + // use primary key as record label field, if it hasn't been set so far // + // todo - does this belong in the enricher?? // + ///////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(qTableMetaData.getRecordLabelFields()) && StringUtils.hasContent(qTableMetaData.getPrimaryKeyField())) + { + qTableMetaData.setRecordLabelFields(List.of(qTableMetaData.getPrimaryKeyField())); + } + + return qTableMetaData; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private MetaDataCustomizerInterface getMetaDataProductionCustomizer(Class> metaDataCustomizerClass) throws QException + { + try + { + return metaDataCustomizerClass.getConstructor().newInstance(); + } + catch(Exception e) + { + throw (new QException("Error constructing table metadata production customizer class [" + metaDataCustomizerClass + "]: ", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void addRecordEntityTableMetaDataProductionCustomizer(MetaDataCustomizerInterface metaDataMetaDataCustomizer) + { + metaDataCustomizers.add(metaDataMetaDataCustomizer); + } + + + + /******************************************************************************* + ** Getter for defaultMetaDataCustomizer + *******************************************************************************/ + public static MetaDataCustomizerInterface getDefaultMetaDataCustomizer() + { + return (RecordEntityToTableGenericMetaDataProducer.defaultMetaDataCustomizer); + } + + + + /******************************************************************************* + ** Setter for defaultMetaDataCustomizer + *******************************************************************************/ + public static void setDefaultMetaDataCustomizer(MetaDataCustomizerInterface defaultMetaDataCustomizer) + { + RecordEntityToTableGenericMetaDataProducer.defaultMetaDataCustomizer = defaultMetaDataCustomizer; + } + + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public RecordEntityToTableGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/ChildRecordListWidget.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/ChildRecordListWidget.java index 9a5299c5..f0952335 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/ChildRecordListWidget.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/ChildRecordListWidget.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.producers.annotations; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; /*************************************************************************** @@ -43,4 +44,6 @@ public @interface ChildRecordListWidget boolean canAddChildRecords() default false; String manageAssociationName() default ""; + + Class widgetMetaDataCustomizer() default MetaDataCustomizerInterface.NoopMetaDataCustomizer.class; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/QMetaDataProducingEntity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/QMetaDataProducingEntity.java index 4e755163..5683baa4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/QMetaDataProducingEntity.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/annotations/QMetaDataProducingEntity.java @@ -26,6 +26,7 @@ import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; /******************************************************************************* @@ -41,8 +42,10 @@ import java.lang.annotation.Target; @SuppressWarnings("checkstyle:MissingJavadocMethod") public @interface QMetaDataProducingEntity { - boolean producePossibleValueSource() default true; + boolean produceTableMetaData() default false; + Class tableMetaDataCustomizer() default MetaDataCustomizerInterface.NoopMetaDataCustomizer.class; + boolean producePossibleValueSource() default false; ChildTable[] childTables() default { }; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java new file mode 100644 index 00000000..3383a4c6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java @@ -0,0 +1,122 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +/*************************************************************************** + ** Common (maybe)? qbit config pattern, where the qbit may be able to provide + ** a particular table, or, the application may supply it itself. + ** + ** If the qbit provides it, then we need to be told (by the application) + ** what backendName to use for the table. + ** + ** Else if the application supplies it, it needs to tell the qBit what the + ** tableName is. + ***************************************************************************/ +public class ProvidedOrSuppliedTableConfig +{ + private boolean doProvideTable; + private String backendName; + private String tableName; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public ProvidedOrSuppliedTableConfig(boolean doProvideTable, String backendName, String tableName) + { + this.doProvideTable = doProvideTable; + this.backendName = backendName; + this.tableName = tableName; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProvidedOrSuppliedTableConfig provideTableUsingBackendNamed(String backendName) + { + return (new ProvidedOrSuppliedTableConfig(true, backendName, null)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProvidedOrSuppliedTableConfig useSuppliedTaleNamed(String tableName) + { + return (new ProvidedOrSuppliedTableConfig(false, null, tableName)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public String getEffectiveTableName(String tableNameIfProviding) + { + if (getDoProvideTable()) + { + return tableNameIfProviding; + } + else + { + return getTableName(); + } + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Getter for doProvideTable + ** + *******************************************************************************/ + public boolean getDoProvideTable() + { + return doProvideTable; + } + + + + /******************************************************************************* + ** Getter for backendName + ** + *******************************************************************************/ + public String getBackendName() + { + return backendName; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java new file mode 100644 index 00000000..346633f8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java @@ -0,0 +1,71 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput; + + +/******************************************************************************* + ** extension of MetaDataProducerInterface, designed for producing meta data + ** within a (java-defined, at this time) QBit. + ** + ** Specifically exists to accept the QBitConfig as a type parameter and a value, + ** easily accessed in the producer's methods as getQBitConfig() + *******************************************************************************/ +public abstract class QBitComponentMetaDataProducer implements MetaDataProducerInterface +{ + private C qBitConfig = null; + + + + /******************************************************************************* + ** Getter for qBitConfig + *******************************************************************************/ + public C getQBitConfig() + { + return (this.qBitConfig); + } + + + + /******************************************************************************* + ** Setter for qBitConfig + *******************************************************************************/ + public void setQBitConfig(C qBitConfig) + { + this.qBitConfig = qBitConfig; + } + + + + /******************************************************************************* + ** Fluent setter for qBitConfig + *******************************************************************************/ + public QBitComponentMetaDataProducer withQBitConfig(C qBitConfig) + { + this.qBitConfig = qBitConfig; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java new file mode 100644 index 00000000..bb4ea5c1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Interface for configuration settings used both in the production of meta-data + ** for a QBit, but also at runtime, e.g., to be aware of exactly how the qbit + ** has been incorporated into an application. + ** + ** For example: + ** - should the QBit define certain tables, or will they be supplied by the application? + ** - what other meta-data names should the qbit reference (backends, schedulers) + ** - what meta-data-customizer(s) should be used? + ** + ** When implementing a QBit, you'll implement this interface - adding whatever + ** (if any) properties you need, and if you have any rules, then overriding + ** the validate method (ideally the one that takes the List-of-String errors) + ** + ** When using a QBit, you'll create an instance of the QBit's config object, + ** and pass it through to the QBit producer. + *******************************************************************************/ +public interface QBitConfig extends Serializable +{ + QLogger LOG = QLogger.getLogger(QBitConfig.class); + + + /*************************************************************************** + ** + ***************************************************************************/ + default void validate(QInstance qInstance) throws QBitConfigValidationException + { + List errors = new ArrayList<>(); + + try + { + validate(qInstance, errors); + } + catch(Exception e) + { + LOG.warn("Error validating QBitConfig: " + this.getClass().getName(), e); + } + + if(!errors.isEmpty()) + { + throw (new QBitConfigValidationException(this, errors)); + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default void validate(QInstance qInstance, List errors) + { + ///////////////////////////////////// + // nothing to validate by default! // + ///////////////////////////////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default boolean assertCondition(boolean condition, String message, List errors) + { + if(!condition) + { + errors.add(message); + } + return (condition); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default MetaDataCustomizerInterface getTableMetaDataCustomizer() + { + return (null); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java new file mode 100644 index 00000000..058442c5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** thrown by QBitConfig.validate() if there's an issue. + *******************************************************************************/ +public class QBitConfigValidationException extends QException +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public QBitConfigValidationException(QBitConfig qBitConfig, List errors) + { + super("Validation failed for QBitConfig: " + qBitConfig.getClass().getName() + ":\n" + StringUtils.join("\n", errors)); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java new file mode 100644 index 00000000..feacb969 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java @@ -0,0 +1,237 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Meta-data to define an active QBit in a QQQ Instance. + ** + ** The unique "name" for the QBit is composed of its groupId and artifactId + ** (maven style). There is also a version - but it is not part of the unique + ** name. But - there is also a namespace attribute, which IS part of the + ** unique name. This will (eventually?) allow us to have multiple instances + ** of the same qbit in a qInstance at the same time (e.g., 2 versions of some + ** table, which should be namespace-prefixed); + ** + ** QBitMetaData also retains the QBitConfig that was used to produce the QBit. + ** + ** Some meta-data objects are aware of the fact that they may have come from a + ** QBit - see SourceQBitAware interface. These objects can get their source + ** QBitMetaData (this object) and its config,via that interface. + *******************************************************************************/ +public class QBitMetaData implements TopLevelMetaDataInterface +{ + private String groupId; + private String artifactId; + private String version; + private String namespace; + + private QBitConfig config; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getName() + { + String name = groupId + ":" + artifactId; + if(StringUtils.hasContent(namespace)) + { + name += ":" + namespace; + } + return name; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addQBit(this); + } + + + + /******************************************************************************* + ** Getter for config + *******************************************************************************/ + public QBitConfig getConfig() + { + return (this.config); + } + + + + /******************************************************************************* + ** Setter for config + *******************************************************************************/ + public void setConfig(QBitConfig config) + { + this.config = config; + } + + + + /******************************************************************************* + ** Fluent setter for config + *******************************************************************************/ + public QBitMetaData withConfig(QBitConfig config) + { + this.config = config; + return (this); + } + + + + /******************************************************************************* + ** Getter for groupId + *******************************************************************************/ + public String getGroupId() + { + return (this.groupId); + } + + + + /******************************************************************************* + ** Setter for groupId + *******************************************************************************/ + public void setGroupId(String groupId) + { + this.groupId = groupId; + } + + + + /******************************************************************************* + ** Fluent setter for groupId + *******************************************************************************/ + public QBitMetaData withGroupId(String groupId) + { + this.groupId = groupId; + return (this); + } + + + + /******************************************************************************* + ** Getter for artifactId + *******************************************************************************/ + public String getArtifactId() + { + return (this.artifactId); + } + + + + /******************************************************************************* + ** Setter for artifactId + *******************************************************************************/ + public void setArtifactId(String artifactId) + { + this.artifactId = artifactId; + } + + + + /******************************************************************************* + ** Fluent setter for artifactId + *******************************************************************************/ + public QBitMetaData withArtifactId(String artifactId) + { + this.artifactId = artifactId; + return (this); + } + + + + /******************************************************************************* + ** Getter for version + *******************************************************************************/ + public String getVersion() + { + return (this.version); + } + + + + /******************************************************************************* + ** Setter for version + *******************************************************************************/ + public void setVersion(String version) + { + this.version = version; + } + + + + /******************************************************************************* + ** Fluent setter for version + *******************************************************************************/ + public QBitMetaData withVersion(String version) + { + this.version = version; + return (this); + } + + + + /******************************************************************************* + ** Getter for namespace + *******************************************************************************/ + public String getNamespace() + { + return (this.namespace); + } + + + + /******************************************************************************* + ** Setter for namespace + *******************************************************************************/ + public void setNamespace(String namespace) + { + this.namespace = namespace; + } + + + + /******************************************************************************* + ** Fluent setter for namespace + *******************************************************************************/ + public QBitMetaData withNamespace(String namespace) + { + this.namespace = namespace; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java new file mode 100644 index 00000000..7fef2174 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** interface for how a QBit's meta-data gets produced and added to a QInstance. + ** + ** When implementing a QBit, you'll implement this interface: + ** - adding a QBitConfig subclass as a property + ** - overriding the produce(qInstance, namespace) method - where you'll: + ** -- create and add your QBitMetaData + ** -- call MetaDataProducerHelper.findProducers + ** -- hand off to finishProducing() in this interface + ** + ** When using a QBit, you'll create an instance of the QBit's config object, + ** pass it in to the producer, then call produce, ala: + ** + ** new SomeQBitProducer() + ** .withQBitConfig(someQBitConfig) + ** .produce(qInstance); + ** + *******************************************************************************/ +public interface QBitProducer +{ + QLogger LOG = QLogger.getLogger(QBitProducer.class); + + + /*************************************************************************** + ** + ***************************************************************************/ + default void produce(QInstance qInstance) throws QException + { + produce(qInstance, null); + } + + /*************************************************************************** + ** + ***************************************************************************/ + void produce(QInstance qInstance, String namespace) throws QException; + + + /*************************************************************************** + * + ***************************************************************************/ + default void finishProducing(QInstance qInstance, QBitMetaData qBitMetaData, C qBitConfig, List> producers) throws QException + { + qBitConfig.validate(qInstance); + + /////////////////////////////// + // todo - move to base class // + /////////////////////////////// + for(MetaDataProducerInterface producer : producers) + { + if(producer instanceof QBitComponentMetaDataProducer) + { + QBitComponentMetaDataProducer qBitComponentMetaDataProducer = (QBitComponentMetaDataProducer) producer; + qBitComponentMetaDataProducer.setQBitConfig(qBitConfig); + } + + if(!producer.isEnabled()) + { + LOG.debug("Not using producer which is not enabled", logPair("producer", producer.getClass().getSimpleName())); + continue; + } + + MetaDataProducerOutput output = producer.produce(qInstance); + + ///////////////////////////////////////// + // apply table customizer, if provided // + ///////////////////////////////////////// + if(qBitConfig.getTableMetaDataCustomizer() != null && output instanceof QTableMetaData table) + { + output = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table); + } + + ///////////////////////////////////////////////// + // set source qbit, if output is aware of such // + ///////////////////////////////////////////////// + if(output instanceof SourceQBitAware sourceQBitAware) + { + sourceQBitAware.setSourceQBitName(qBitMetaData.getName()); + } + + output.addSelfToInstance(qInstance); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java new file mode 100644 index 00000000..c4d4be7f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.context.QContext; + + +/******************************************************************************* + ** interface for meta data objects that may have come from a qbit, and where we + ** might want to get data about that qbit (e.g., config or meta-data). + *******************************************************************************/ +public interface SourceQBitAware +{ + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + String getSourceQBitName(); + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + void setSourceQBitName(String sourceQBitName); + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + Object withSourceQBitName(String sourceQBitName); + + + /*************************************************************************** + ** + ***************************************************************************/ + default QBitMetaData getSourceQBit() + { + String qbitName = getSourceQBitName(); + return (QContext.getQInstance().getQBits().get(qbitName)); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default QBitConfig getSourceQBitConfig() + { + QBitMetaData sourceQBit = getSourceQBit(); + if(sourceQBit == null) + { + return null; + } + else + { + return sourceQBit.getConfig(); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index e6078562..b2318de5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -50,10 +50,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -61,25 +63,19 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; ** Meta-Data to define a table in a QQQ instance. ** *******************************************************************************/ -public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface +public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface, SourceQBitAware { private static final QLogger LOG = QLogger.getLogger(QTableMetaData.class); private String name; private String label; - // TODO: resolve confusion over: - // Is this name of what backend the table is stored in (yes) - // Or the "name" of the table WITHIN the backend (no) - // although that's how "backendName" is used in QFieldMetaData. - // Idea: - // rename "backendName" here to "backend" - // add "nameInBackend" (or similar) for the table name in the backend - // OR - add a whole "backendDetails" object, with different details per backend-type private String backendName; private String primaryKeyField; private boolean isHidden = false; + private String sourceQBitName; + private Map fields; private List uniqueKeys; private List associations; @@ -184,6 +180,12 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData } } + /////////////////////////////////////////////////////////////////////////////////////////////////// + // stash a reference from this entityClass to this table in the QRecordEntity class // + // (used within that class later, if it wants to know about a table that an Entity helped build) // + /////////////////////////////////////////////////////////////////////////////////////////////////// + QRecordEntity.registerTable(entityClass, this); + return (this); } @@ -714,6 +716,25 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData + /******************************************************************************* + ** Getter for sections + ** + *******************************************************************************/ + public QFieldSection getSection(String name) + { + for(QFieldSection qFieldSection : CollectionUtils.nonNullList(sections)) + { + if(qFieldSection.getName().equals(name)) + { + return (qFieldSection); + } + } + + return (null); + } + + + /******************************************************************************* ** Setter for sections ** @@ -1038,7 +1059,7 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData { for(Capability disabledCapability : disabledCapabilities) { - withCapability(disabledCapability); + withoutCapability(disabledCapability); } return (this); } @@ -1536,4 +1557,38 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot); } + + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + @Override + public QTableMetaData withSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java new file mode 100644 index 00000000..52f84d78 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java @@ -0,0 +1,221 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; + + +/******************************************************************************* + ** Factory class for creating "standard" qfield sections. e.g., if you want + ** the same t1, t2, and t3 section on all your tables, use this class to + ** produce them. + ** + ** You can change the default name & iconNames for those sections, but note, + ** this is a static/utility style class, so those settings are static fields. + ** + ** The method customT2 is provided as not much of a shortcut over "doing it yourself", + ** but to allow all sections for a table to be produced through calls to this factory, + ** so they look more similar. + *******************************************************************************/ +public class SectionFactory +{ + private static String defaultT1name = "identity"; + private static String defaultT1iconName = "badge"; + private static String defaultT2name = "data"; + private static String defaultT2iconName = "text_snippet"; + private static String defaultT3name = "dates"; + private static String defaultT3iconName = "calendar_month"; + + + /******************************************************************************* + ** private constructor, to enforce static usage, e.g., to make clear the fields + ** are static fields. + ** + *******************************************************************************/ + private SectionFactory() + { + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT1(String... fieldNames) + { + return new QFieldSection(defaultT1name, new QIcon().withName(defaultT1iconName), Tier.T1, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT2(String... fieldNames) + { + return new QFieldSection(defaultT2name, new QIcon().withName(defaultT2iconName), Tier.T2, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection customT2(String name, QIcon icon, String... fieldNames) + { + return new QFieldSection(name, icon, Tier.T2, List.of(fieldNames)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QFieldSection defaultT3(String... fieldNames) + { + return new QFieldSection(defaultT3name, new QIcon().withName(defaultT3iconName), Tier.T3, List.of(fieldNames)); + } + + + + /******************************************************************************* + ** Getter for defaultT1name + *******************************************************************************/ + public static String getDefaultT1name() + { + return (SectionFactory.defaultT1name); + } + + + + /******************************************************************************* + ** Setter for defaultT1name + *******************************************************************************/ + public static void setDefaultT1name(String defaultT1name) + { + SectionFactory.defaultT1name = defaultT1name; + } + + + + /******************************************************************************* + ** Getter for defaultT1iconName + *******************************************************************************/ + public static String getDefaultT1iconName() + { + return (SectionFactory.defaultT1iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT1iconName + *******************************************************************************/ + public static void setDefaultT1iconName(String defaultT1iconName) + { + SectionFactory.defaultT1iconName = defaultT1iconName; + } + + + + /******************************************************************************* + ** Getter for defaultT2name + *******************************************************************************/ + public static String getDefaultT2name() + { + return (SectionFactory.defaultT2name); + } + + + + /******************************************************************************* + ** Setter for defaultT2name + *******************************************************************************/ + public static void setDefaultT2name(String defaultT2name) + { + SectionFactory.defaultT2name = defaultT2name; + } + + + + /******************************************************************************* + ** Getter for defaultT2iconName + *******************************************************************************/ + public static String getDefaultT2iconName() + { + return (SectionFactory.defaultT2iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT2iconName + *******************************************************************************/ + public static void setDefaultT2iconName(String defaultT2iconName) + { + SectionFactory.defaultT2iconName = defaultT2iconName; + } + + + + /******************************************************************************* + ** Getter for defaultT3name + *******************************************************************************/ + public static String getDefaultT3name() + { + return (SectionFactory.defaultT3name); + } + + + + /******************************************************************************* + ** Setter for defaultT3name + *******************************************************************************/ + public static void setDefaultT3name(String defaultT3name) + { + SectionFactory.defaultT3name = defaultT3name; + } + + + + /******************************************************************************* + ** Getter for defaultT3iconName + *******************************************************************************/ + public static String getDefaultT3iconName() + { + return (SectionFactory.defaultT3iconName); + } + + + + /******************************************************************************* + ** Setter for defaultT3iconName + *******************************************************************************/ + public static void setDefaultT3iconName(String defaultT3iconName) + { + SectionFactory.defaultT3iconName = defaultT3iconName; + } + + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java new file mode 100644 index 00000000..e72d188f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** possible-value source provider for the `Tables` PVS - a list of all tables + ** in an application/qInstance. + *******************************************************************************/ +public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue)); + if(table != null && !table.getIsHidden()) + { + PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table); + if(PermissionCheckResult.ALLOW.equals(permissionCheckResult)) + { + return (new QPossibleValue<>(table.getName(), table.getLabel())); + } + } + + return null; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////// + // build all of the possible values (note, will be filtered by user's permissions) // + ///////////////////////////////////////////////////////////////////////////////////// + List> allPossibleValues = new ArrayList<>(); + for(QTableMetaData table : QContext.getQInstance().getTables().values()) + { + QPossibleValue possibleValue = getPossibleValue(table.getName()); + if(possibleValue != null) + { + allPossibleValues.add(possibleValue); + } + } + + return completeCustomPVSSearch(input, allPossibleValues); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java index 27cce6df..75087ae7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java @@ -22,17 +22,11 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; -import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -51,22 +45,10 @@ public class TablesPossibleValueSourceMetaDataProvider { QPossibleValueSource possibleValueSource = new QPossibleValueSource() .withName(NAME) - .withType(QPossibleValueSourceType.ENUM) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class)) .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY); - List> enumValues = new ArrayList<>(); - for(QTableMetaData table : qInstance.getTables().values()) - { - if(BooleanUtils.isNotTrue(table.getIsHidden())) - { - String label = StringUtils.hasContent(table.getLabel()) ? table.getLabel() : QInstanceEnricher.nameToLabel(table.getName()); - enumValues.add(new QPossibleValue<>(table.getName(), label)); - } - } - - enumValues.sort(Comparator.comparing(QPossibleValue::getLabel)); - - possibleValueSource.withEnumValues(enumValues); return (possibleValueSource); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java new file mode 100644 index 00000000..ea57a3ed --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java @@ -0,0 +1,31 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +/******************************************************************************* + ** interface to be implemented by enums (presumably) that define the possible + ** settings a particular backend type can get from a variant record. + *******************************************************************************/ +public interface BackendVariantSetting +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java new file mode 100644 index 00000000..290c099f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java @@ -0,0 +1,225 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.util.HashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** Configs for how a backend that uses variants works. Specifically: + ** + ** - the variant "type key" - e.g., key for variants map in session. + ** - what table supplies the variant options (optionsTableName + ** - an optional filter to apply to that options table + ** - a map of the settings that a backend gets from its variant table to the + ** field names in that table that they come from. e.g., a backend may have a + ** username attribute, whose value comes from a field named "theUser" in the + ** variant options table. + ** - an optional code reference to a variantRecordLookupFunction - to customize + ** how the variant record is looked up (such as, adding joined or other custom + ** fields). + *******************************************************************************/ +public class BackendVariantsConfig +{ + private String variantTypeKey; + + private String optionsTableName; + private QQueryFilter optionsFilter; + private QCodeReference variantRecordLookupFunction; + + private Map backendSettingSourceFieldNameMap; + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getOptionsTableName() + { + return (this.optionsTableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setOptionsTableName(String optionsTableName) + { + this.optionsTableName = optionsTableName; + } + + + + /******************************************************************************* + ** Getter for filter + *******************************************************************************/ + public QQueryFilter getOptionsFilter() + { + return (this.optionsFilter); + } + + + + /******************************************************************************* + ** Setter for filter + *******************************************************************************/ + public void setOptionsFilter(QQueryFilter optionsFilter) + { + this.optionsFilter = optionsFilter; + } + + + + /******************************************************************************* + ** Getter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public Map getBackendSettingSourceFieldNameMap() + { + return (this.backendSettingSourceFieldNameMap); + } + + + + /******************************************************************************* + ** Setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public void setBackendSettingSourceFieldNameMap(Map backendSettingSourceFieldNameMap) + { + this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap; + } + + + + /******************************************************************************* + ** Fluent setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public BackendVariantsConfig withBackendSettingSourceFieldName(BackendVariantSetting backendVariantSetting, String sourceFieldName) + { + if(this.backendSettingSourceFieldNameMap == null) + { + this.backendSettingSourceFieldNameMap = new HashMap<>(); + } + this.backendSettingSourceFieldNameMap.put(backendVariantSetting, sourceFieldName); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for backendSettingSourceFieldNameMap + *******************************************************************************/ + public BackendVariantsConfig withBackendSettingSourceFieldNameMap(Map backendSettingSourceFieldNameMap) + { + this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for variantTypeKey + *******************************************************************************/ + public String getVariantTypeKey() + { + return (this.variantTypeKey); + } + + + + /******************************************************************************* + ** Setter for variantTypeKey + *******************************************************************************/ + public void setVariantTypeKey(String variantTypeKey) + { + this.variantTypeKey = variantTypeKey; + } + + + + /******************************************************************************* + ** Fluent setter for variantTypeKey + *******************************************************************************/ + public BackendVariantsConfig withVariantTypeKey(String variantTypeKey) + { + this.variantTypeKey = variantTypeKey; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for optionsTableName + *******************************************************************************/ + public BackendVariantsConfig withOptionsTableName(String optionsTableName) + { + this.optionsTableName = optionsTableName; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for optionsFilter + *******************************************************************************/ + public BackendVariantsConfig withOptionsFilter(QQueryFilter optionsFilter) + { + this.optionsFilter = optionsFilter; + return (this); + } + + + /******************************************************************************* + ** Getter for variantRecordLookupFunction + *******************************************************************************/ + public QCodeReference getVariantRecordLookupFunction() + { + return (this.variantRecordLookupFunction); + } + + + + /******************************************************************************* + ** Setter for variantRecordLookupFunction + *******************************************************************************/ + public void setVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction) + { + this.variantRecordLookupFunction = variantRecordLookupFunction; + } + + + + /******************************************************************************* + ** Fluent setter for variantRecordLookupFunction + *******************************************************************************/ + public BackendVariantsConfig withVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction) + { + this.variantRecordLookupFunction = variantRecordLookupFunction; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java new file mode 100644 index 00000000..a94fbc4e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.io.Serializable; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; + + +/******************************************************************************* + ** Utility methods for backends working with Variants. + *******************************************************************************/ +public class BackendVariantsUtil +{ + + /******************************************************************************* + ** Get the variant id from the session for the backend. + *******************************************************************************/ + public static Serializable getVariantId(QBackendMetaData backendMetaData) throws QException + { + QSession session = QContext.getQSession(); + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) + { + throw (new QException("Could not find Backend Variant information in session under key '" + variantTypeKey + "' for Backend '" + backendMetaData.getName() + "'")); + } + Serializable variantId = session.getBackendVariants().get(variantTypeKey); + return variantId; + } + + + + /******************************************************************************* + ** For backends that use variants, look up the variant record (in theory, based + ** on an id in the session's backend variants map, then fetched from the backend's + ** variant options table. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException + { + Serializable variantId = getVariantId(backendMetaData); + + QRecord record; + if(backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction() != null) + { + Object o = QCodeLoader.getAdHoc(Object.class, backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction()); + if(o instanceof UnsafeFunction unsafeFunction) + { + record = ((UnsafeFunction) unsafeFunction).apply(variantId); + } + else if(o instanceof Function function) + { + record = ((Function) function).apply(variantId); + } + else + { + throw (new QException("Backend Variant's recordLookupFunction is not of any expected type (should have been caught by instance validation??)")); + } + } + else + { + GetInput getInput = new GetInput(); + getInput.setShouldMaskPasswords(false); + getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + getInput.setPrimaryKey(variantId); + GetOutput getOutput = new GetAction().execute(getInput); + + record = getOutput.getRecord(); + } + + if(record == null) + { + throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); + } + return record; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java new file mode 100644 index 00000000..59ccc38c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +/******************************************************************************* + ** temporary class, while we migrate from original way that variants were set up + ** e.g., by calling 'variantOptionsTableUsernameField', to the new way, using + ** the BackendVariantsConfig which uses a map of enum constants. + ** + ** so when those deprecated setters are removed, this enum can be too. + *****************************************************************************/ +public enum LegacyBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + API_KEY, + CLIENT_ID, + CLIENT_SECRET +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcess.java new file mode 100644 index 00000000..6702fca1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcess.java @@ -0,0 +1,230 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.processes; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** QRecord Entity for QQQProcess table - e.g., table that stores an id, name + ** and the label for all processes in the QQQ application. Useful as a foreign + ** key from other logging type tables. + *******************************************************************************/ +public class QQQProcess extends QRecordEntity +{ + public static final String TABLE_NAME = "qqqProcess"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String name; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS) + private String label; + + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public QQQProcess() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public QQQProcess(QRecord record) + { + populateFromQRecord(record); + } + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public QQQProcess withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public QQQProcess withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public QQQProcess withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public QQQProcess withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public QQQProcess withLabel(String label) + { + this.label = label; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessTableManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessTableManager.java new file mode 100644 index 00000000..b7d5c400 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessTableManager.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility class for accessing QQQProcess records (well, just their ids at this time) + ** Takes care of inserting upon a miss, and dealing with the cache table. + *******************************************************************************/ +public class QQQProcessTableManager +{ + private static final QLogger LOG = QLogger.getLogger(QQQProcessTableManager.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Integer getQQQProcessId(QInstance qInstance, String processName) throws QException + { + ///////////////////////////// + // look in the cache table // + ///////////////////////////// + GetInput getInput = new GetInput(); + getInput.setTableName(QQQProcessesMetaDataProvider.QQQ_PROCESS_CACHE_TABLE_NAME); + getInput.setUniqueKey(MapBuilder.of("name", processName)); + GetOutput getOutput = new GetAction().execute(getInput); + + //////////////////////// + // upon cache miss... // + //////////////////////// + if(getOutput.getRecord() == null) + { + QProcessMetaData processMetaData = qInstance.getProcess(processName); + if(processMetaData == null) + { + LOG.info("No such process", logPair("processName", processName)); + return (null); + } + + /////////////////////////////////////////////////////// + // insert the record (into the table, not the cache) // + /////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(QQQProcess.TABLE_NAME); + insertInput.setRecords(List.of(new QRecord().withValue("name", processName).withValue("label", processMetaData.getLabel()))); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + + /////////////////////////////////// + // repeat the get from the cache // + /////////////////////////////////// + getOutput = new GetAction().execute(getInput); + } + + return getOutput.getRecord().getValueInteger("id"); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessesMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessesMetaDataProvider.java new file mode 100644 index 00000000..6cca0f6e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessesMetaDataProvider.java @@ -0,0 +1,134 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.processes; + + +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf; +import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase; + + +/******************************************************************************* + ** Provides meta data for the QQQProcess table, PVS, and a cache table. + *******************************************************************************/ +public class QQQProcessesMetaDataProvider +{ + public static final String QQQ_PROCESS_CACHE_TABLE_NAME = QQQProcess.TABLE_NAME + "Cache"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String persistentBackendName, String cacheBackendName, Consumer backendDetailEnricher) throws QException + { + instance.addTable(defineQQQProcess(persistentBackendName, backendDetailEnricher)); + instance.addTable(defineQQQProcessCache(cacheBackendName, backendDetailEnricher)); + instance.addPossibleValueSource(defineQQQProcessPossibleValueSource()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineQQQProcess(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(QQQProcess.TABLE_NAME) + .withLabel("Process") + .withBackendName(backendName) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE)) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withPrimaryKeyField("id") + .withUniqueKey(new UniqueKey("name")) + .withFieldsFromEntity(QQQProcess.class) + .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineQQQProcessCache(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(QQQ_PROCESS_CACHE_TABLE_NAME) + .withBackendName(backendName) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE)) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withPrimaryKeyField("id") + .withUniqueKey(new UniqueKey("name")) + .withFieldsFromEntity(QQQProcess.class) + .withCacheOf(new CacheOf() + .withSourceTable(QQQProcess.TABLE_NAME) + .withUseCase(new CacheUseCase() + .withType(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY) + .withCacheSourceMisses(false) + .withCacheUniqueKey(new UniqueKey("name")) + .withSourceUniqueKey(new UniqueKey("name")) + .withDoCopySourcePrimaryKeyToCache(true) + ) + ); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QPossibleValueSource defineQQQProcessPossibleValueSource() + { + return (new QPossibleValueSource() + .withType(QPossibleValueSourceType.TABLE) + .withName(QQQProcess.TABLE_NAME) + .withTableName(QQQProcess.TABLE_NAME)) + .withOrderByField("label"); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java new file mode 100644 index 00000000..65f07c77 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java @@ -0,0 +1,285 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; + + +/******************************************************************************* + ** Entity bean for the savedBulkLoadProfile table + *******************************************************************************/ +public class SavedBulkLoadProfile extends QRecordEntity +{ + public static final String TABLE_NAME = "savedBulkLoadProfile"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS, label = "Profile Name") + private String label; + + @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, label = "Table", isRequired = true) + private String tableName; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID, label = "Owner") + private String userId; + + @QField(label = "Mapping JSON") + private String mappingJson; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedBulkLoadProfile() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedBulkLoadProfile(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Setter for id + ** + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Getter for createDate + ** + *******************************************************************************/ + public Instant getCreateDate() + { + return createDate; + } + + + + /******************************************************************************* + ** Setter for createDate + ** + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Getter for modifyDate + ** + *******************************************************************************/ + public Instant getModifyDate() + { + return modifyDate; + } + + + + /******************************************************************************* + ** Setter for modifyDate + ** + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public SavedBulkLoadProfile withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Setter for tableName + ** + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + ** + *******************************************************************************/ + public SavedBulkLoadProfile withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + ** + *******************************************************************************/ + public String getUserId() + { + return userId; + } + + + + /******************************************************************************* + ** Setter for userId + ** + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + ** + *******************************************************************************/ + public SavedBulkLoadProfile withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + + /******************************************************************************* + ** Getter for mappingJson + *******************************************************************************/ + public String getMappingJson() + { + return (this.mappingJson); + } + + + + /******************************************************************************* + ** Setter for mappingJson + *******************************************************************************/ + public void setMappingJson(String mappingJson) + { + this.mappingJson = mappingJson; + } + + + + /******************************************************************************* + ** Fluent setter for mappingJson + *******************************************************************************/ + public SavedBulkLoadProfile withMappingJson(String mappingJson) + { + this.mappingJson = mappingJson; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java new file mode 100644 index 00000000..843a1063 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedBulkLoadProfileJsonFieldDisplayValueFormatter implements FieldDisplayBehavior +{ + private static SavedBulkLoadProfileJsonFieldDisplayValueFormatter savedReportJsonFieldDisplayValueFormatter = null; + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private SavedBulkLoadProfileJsonFieldDisplayValueFormatter() + { + + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static SavedBulkLoadProfileJsonFieldDisplayValueFormatter getInstance() + { + if(savedReportJsonFieldDisplayValueFormatter == null) + { + savedReportJsonFieldDisplayValueFormatter = new SavedBulkLoadProfileJsonFieldDisplayValueFormatter(); + } + return (savedReportJsonFieldDisplayValueFormatter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SavedBulkLoadProfileJsonFieldDisplayValueFormatter getDefault() + { + return getInstance(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + if(field.getName().equals("mappingJson")) + { + String mappingJson = record.getValueString("mappingJson"); + if(StringUtils.hasContent(mappingJson)) + { + try + { + record.setDisplayValue("mappingJson", jsonToDisplayValue(mappingJson)); + } + catch(Exception e) + { + record.setDisplayValue("mappingJson", "Invalid Mapping..."); + } + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String jsonToDisplayValue(String mappingJson) + { + JSONObject jsonObject = new JSONObject(mappingJson); + + List parts = new ArrayList<>(); + + if(jsonObject.has("fieldList")) + { + JSONArray fieldListArray = jsonObject.getJSONArray("fieldList"); + parts.add(fieldListArray.length() + " field" + StringUtils.plural(fieldListArray.length())); + } + + if(jsonObject.has("hasHeaderRow")) + { + boolean hasHeaderRow = jsonObject.getBoolean("hasHeaderRow"); + parts.add((hasHeaderRow ? "With" : "Without") + " header row"); + } + + if(jsonObject.has("layout")) + { + String layout = jsonObject.getString("layout"); + parts.add("Layout: " + StringUtils.allCapsToMixedCase(layout)); + } + + return StringUtils.join("; ", parts); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java new file mode 100644 index 00000000..1438dd37 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.DeleteSavedBulkLoadProfileProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.QuerySavedBulkLoadProfileProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.StoreSavedBulkLoadProfileProcess; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedBulkLoadProfileMetaDataProvider +{ + public static final String SHARED_SAVED_BULK_LOAD_PROFILE_JOIN_SAVED_BULK_LOAD_PROFILE = "sharedSavedBulkLoadProfileJoinSavedBulkLoadProfile"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String recordTablesBackendName, Consumer backendDetailEnricher) throws QException + { + instance.addTable(defineSavedBulkLoadProfileTable(recordTablesBackendName, backendDetailEnricher)); + instance.addPossibleValueSource(QPossibleValueSource.newForTable(SavedBulkLoadProfile.TABLE_NAME)); + + ///////////////////////////////////// + // todo - param to enable sharing? // + ///////////////////////////////////// + instance.addTable(defineSharedSavedBulkLoadProfileTable(recordTablesBackendName, backendDetailEnricher)); + instance.addJoin(defineSharedSavedBulkLoadProfileJoinSavedBulkLoadProfile()); + if(instance.getPossibleValueSource(ShareScopePossibleValueMetaDataProducer.NAME) == null) + { + instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance())); + } + + //////////////////////////////////// + // processes for working with 'em // + //////////////////////////////////// + instance.add(StoreSavedBulkLoadProfileProcess.getProcessMetaData()); + instance.add(QuerySavedBulkLoadProfileProcess.getProcessMetaData()); + instance.add(DeleteSavedBulkLoadProfileProcess.getProcessMetaData()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QJoinMetaData defineSharedSavedBulkLoadProfileJoinSavedBulkLoadProfile() + { + return (new QJoinMetaData() + .withName(SHARED_SAVED_BULK_LOAD_PROFILE_JOIN_SAVED_BULK_LOAD_PROFILE) + .withLeftTable(SharedSavedBulkLoadProfile.TABLE_NAME) + .withRightTable(SavedBulkLoadProfile.TABLE_NAME) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("savedBulkLoadProfileId", "id"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSavedBulkLoadProfileTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SavedBulkLoadProfile.TABLE_NAME) + .withLabel("Bulk Load Profile") + .withIcon(new QIcon().withName("drive_folder_upload")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SavedBulkLoadProfile.class) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) + .withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + table.getField("mappingJson").withBehavior(SavedBulkLoadProfileJsonFieldDisplayValueFormatter.getInstance()); + table.getField("mappingJson").setLabel("Mapping"); + + table.withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(SharedSavedBulkLoadProfile.TABLE_NAME) + .withAssetIdFieldName("savedBulkLoadProfileId") + .withScopeFieldName("scope") + .withThisTableOwnerIdFieldName("userId") + .withAudienceType(new ShareableAudienceType().withName("user").withFieldName("userId"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSharedSavedBulkLoadProfileTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SharedSavedBulkLoadProfile.TABLE_NAME) + .withLabel("Shared Bulk Load Profile") + .withIcon(new QIcon().withName("share")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("savedBulkLoadProfileId") + .withBackendName(backendName) + .withUniqueKey(new UniqueKey("savedBulkLoadProfileId", "userId")) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SharedSavedBulkLoadProfile.class) + // todo - security key + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedBulkLoadProfileId", "userId"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("scope"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java new file mode 100644 index 00000000..a02e5c3b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java @@ -0,0 +1,268 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; + + +/******************************************************************************* + ** Entity bean for the shared saved bulk load profile table + *******************************************************************************/ +public class SharedSavedBulkLoadProfile extends QRecordEntity +{ + public static final String TABLE_NAME = "sharedSavedBulkLoadProfile"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = SavedBulkLoadProfile.TABLE_NAME, label = "Bulk Load Profile") + private Integer savedBulkLoadProfileId; + + @QField(label = "User") + private String userId; + + @QField(possibleValueSourceName = ShareScopePossibleValueMetaDataProducer.NAME) + private String scope; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedBulkLoadProfile() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedBulkLoadProfile(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public SharedSavedBulkLoadProfile withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public SharedSavedBulkLoadProfile withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public SharedSavedBulkLoadProfile withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public SharedSavedBulkLoadProfile withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for scope + *******************************************************************************/ + public String getScope() + { + return (this.scope); + } + + + + /******************************************************************************* + ** Setter for scope + *******************************************************************************/ + public void setScope(String scope) + { + this.scope = scope; + } + + + + /******************************************************************************* + ** Fluent setter for scope + *******************************************************************************/ + public SharedSavedBulkLoadProfile withScope(String scope) + { + this.scope = scope; + return (this); + } + + + + /******************************************************************************* + ** Getter for savedBulkLoadProfileId + *******************************************************************************/ + public Integer getSavedBulkLoadProfileId() + { + return (this.savedBulkLoadProfileId); + } + + + + /******************************************************************************* + ** Setter for savedBulkLoadProfileId + *******************************************************************************/ + public void setSavedBulkLoadProfileId(Integer savedBulkLoadProfileId) + { + this.savedBulkLoadProfileId = savedBulkLoadProfileId; + } + + + + /******************************************************************************* + ** Fluent setter for savedBulkLoadProfileId + *******************************************************************************/ + public SharedSavedBulkLoadProfile withSavedBulkLoadProfileId(Integer savedBulkLoadProfileId) + { + this.savedBulkLoadProfileId = savedBulkLoadProfileId; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java index fc90fb1f..f71da271 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; -import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.context.QContext; @@ -43,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.DynamicFormWidgetData; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAndJoinTable; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.SavedReportToReportMetaDataAdapter; @@ -56,7 +56,16 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** Note - exists under 2 names, for the RenderSavedReport process, and for the - ** ScheduledReport table + ** ScheduledReport table (and can be used in your custom code too: + * + ** by default, in qqq backend core, we'll assume this widget is being used on the + ** view screen for a ScheduledReport, with field names that we know from that table. + ** But, allow it to be used on a different table (optionally with different field names), + ** coming from the input map. + ** + ** e.g., that one may set in widget metaData as: + ** .withDefaultValue("tableName", "myTable") + ** .withDefaultValue("fieldNameId", "identifier"), etc. *******************************************************************************/ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRenderer { @@ -88,11 +97,16 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere } else if(input.getQueryParams().containsKey("id")) { - QRecord scheduledReportRecord = new GetAction().executeForRecord(new GetInput(ScheduledReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get("id")))); - QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(scheduledReportRecord.getValueInteger("savedReportId")))); + String tableName = input.getQueryParams().getOrDefault("tableName", ScheduledReport.TABLE_NAME); + String fieldNameId = input.getQueryParams().getOrDefault("fieldNameId", "id"); + String fieldNameSavedReportId = input.getQueryParams().getOrDefault("fieldNameSavedReportId", "savedReportId"); + String fieldNameInputValues = input.getQueryParams().getOrDefault("fieldNameInputValues", "inputValues"); + + QRecord hostRecord = new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get(fieldNameId)))); + QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(hostRecord.getValueInteger(fieldNameSavedReportId)))); savedReport = new SavedReport(record); - String inputValues = scheduledReportRecord.getValueString("inputValues"); + String inputValues = hostRecord.getValueString(fieldNameInputValues); if(StringUtils.hasContent(inputValues)) { JSONObject jsonObject = JsonUtils.toJSONObject(inputValues); @@ -127,8 +141,8 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere { if(criteriaValue instanceof FilterVariableExpression filterVariableExpression) { - GenerateReportAction.FieldAndJoinTable fieldAndJoinTable = GenerateReportAction.getFieldAndJoinTable(table, criteria.getFieldName()); - QFieldMetaData fieldMetaData = fieldAndJoinTable.field().clone(); + FieldAndJoinTable fieldAndJoinTable = FieldAndJoinTable.get(table, criteria.getFieldName()); + QFieldMetaData fieldMetaData = fieldAndJoinTable.field().clone(); ///////////////////////////////// // make name & label for field // @@ -197,8 +211,8 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere } catch(Exception e) { - LOG.warn("Error rendering scheduled report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams()))); - throw (new QException("Error rendering scheduled report values dynamic form widget", e)); + LOG.warn("Error rendering report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams()))); + throw (new QException("Error rendering report values dynamic form widget", e)); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java index 6fe6e9bf..65e11a10 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; -import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; @@ -39,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAndJoinTable; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage; @@ -311,7 +311,7 @@ public class SavedReportTableCustomizer implements TableCustomizerInterface { try { - GenerateReportAction.FieldAndJoinTable fieldAndJoinTable = GenerateReportAction.getFieldAndJoinTable(table, fieldName); + FieldAndJoinTable fieldAndJoinTable = FieldAndJoinTable.get(table, fieldName); return (fieldAndJoinTable.getLabel(table)); } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java index 2b40bd86..13084a42 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java @@ -50,7 +50,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -351,8 +350,7 @@ public class SavedReportsMetaDataProvider .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId", "renderedReportStatusId"))) .withSection(new QFieldSection("input", new QIcon().withName("input"), Tier.T2, List.of("userId", "reportFormat"))) .withSection(new QFieldSection("output", new QIcon().withName("output"), Tier.T2, List.of("jobUuid", "resultPath", "rowCount", "errorMessage", "startTime", "endTime"))) - .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) - .withoutCapabilities(Capability.allWriteCapabilities()); + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); table.getField("renderedReportStatusId").setAdornments(List.of(new FieldAdornment(AdornmentType.CHIP) .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.RUNNING.getId(), "pending", AdornmentType.ChipValues.COLOR_SECONDARY)) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java index cf050a18..284942ba 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java @@ -48,7 +48,7 @@ public class QSession implements Serializable, Cloneable private QUser user; private String uuid; - private Set permissions; + private Set permissions; private Map> securityKeyValues; private Map backendVariants; @@ -360,12 +360,38 @@ public class QSession implements Serializable, Cloneable return (false); } - List values = securityKeyValues.get(keyName); - Serializable valueAsType = ValueUtils.getValueAsFieldType(fieldType, value); + List values = securityKeyValues.get(keyName); + + Serializable valueAsType; + try + { + valueAsType = ValueUtils.getValueAsFieldType(fieldType, value); + } + catch(Exception e) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // an exception in getValueAsFieldType would indicate, e.g., a non-number string trying to come back as integer. // + // so - assume that any such mismatch means the value isn't in the session. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (false); + } + for(Serializable keyValue : values) { - Serializable keyValueAsType = ValueUtils.getValueAsFieldType(fieldType, keyValue); - if(keyValueAsType.equals(valueAsType)) + Serializable keyValueAsType = null; + try + { + keyValueAsType = ValueUtils.getValueAsFieldType(fieldType, keyValue); + } + catch(Exception e) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // an exception in getValueAsFieldType would indicate, e.g., a non-number string trying to come back as integer. // + // so - assume that any such mismatch means this key isn't a match. + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + } + + if(valueAsType.equals(keyValueAsType)) { return (true); } @@ -561,6 +587,7 @@ public class QSession implements Serializable, Cloneable } + /******************************************************************************* ** Getter for valuesForFrontend *******************************************************************************/ @@ -591,6 +618,7 @@ public class QSession implements Serializable, Cloneable } + /******************************************************************************* ** Fluent setter for a single valuesForFrontend *******************************************************************************/ @@ -604,5 +632,4 @@ public class QSession implements Serializable, Cloneable return (this); } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTable.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTable.java index b0b607bf..2fe0b4fb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTable.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTable.java @@ -30,7 +30,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior /******************************************************************************* - ** QRecord Entity for QQQTable table + ** QRecord Entity for QQQTable table - e.g., table that stores an id, name + ** and the label for all tables in the QQQ application. Useful as a foreign + ** key from other logging type tables. *******************************************************************************/ public class QQQTable extends QRecordEntity { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTableTableManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTableTableManager.java new file mode 100644 index 00000000..d15aabb7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTableTableManager.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility class for accessing QQQTable records (well, just their ids at this time) + ** Takes care of inserting upon a miss, and dealing with the cache table. + *******************************************************************************/ +public class QQQTableTableManager +{ + private static final QLogger LOG = QLogger.getLogger(QQQTableTableManager.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Integer getQQQTableId(QInstance qInstance, String tableName) throws QException + { + ///////////////////////////// + // look in the cache table // + ///////////////////////////// + GetInput getInput = new GetInput(); + getInput.setTableName(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME); + getInput.setUniqueKey(MapBuilder.of("name", tableName)); + GetOutput getOutput = new GetAction().execute(getInput); + + //////////////////////// + // upon cache miss... // + //////////////////////// + if(getOutput.getRecord() == null) + { + QTableMetaData tableMetaData = qInstance.getTable(tableName); + if(tableMetaData == null) + { + LOG.info("No such table", logPair("tableName", tableName)); + return (null); + } + + /////////////////////////////////////////////////////// + // insert the record (into the table, not the cache) // + /////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(QQQTable.TABLE_NAME); + insertInput.setRecords(List.of(new QRecord().withValue("name", tableName).withValue("label", tableMetaData.getLabel()))); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + + /////////////////////////////////// + // repeat the get from the cache // + /////////////////////////////////// + getOutput = new GetAction().execute(getInput); + } + + return getOutput.getRecord().getValueInteger("id"); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java index 034efcd5..e880a230 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java @@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase; /******************************************************************************* - ** + ** Provides meta data for the QQQTable table, PVS, and a cache table. *******************************************************************************/ public class QQQTablesMetaDataProvider { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 7591becb..b53d69a2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -193,7 +193,7 @@ public class MemoryRecordStore if(recordMatches) { qRecord.setErrors(new ArrayList<>()); - ValidateRecordSecurityLockHelper.validateSecurityFields(input.getTable(), List.of(qRecord), ValidateRecordSecurityLockHelper.Action.SELECT); + ValidateRecordSecurityLockHelper.validateSecurityFields(input.getTable(), List.of(qRecord), ValidateRecordSecurityLockHelper.Action.SELECT, null); if(CollectionUtils.nullSafeHasContents(qRecord.getErrors())) { ////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java index 281ee9cc..e59038e0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java @@ -34,6 +34,7 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaOption; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; @@ -268,6 +269,11 @@ public class BackendQueryFilterUtils String regex = sqlLikeToRegex(criterionValue); + if(criterion.hasOption(CriteriaOption.CASE_INSENSITIVE)) + { + return (stringValue.toLowerCase().matches(regex.toLowerCase())); + } + return (stringValue.matches(regex)); } @@ -427,6 +433,23 @@ public class BackendQueryFilterUtils } } + if(criterion.hasOption(CriteriaOption.CASE_INSENSITIVE)) + { + if(CollectionUtils.nullSafeHasContents(criterion.getValues())) + { + if(criterion.getValues().get(0) instanceof String) + { + for(Serializable criterionValue : criterion.getValues()) + { + if(criterionValue instanceof String criterionValueString && value instanceof String valueString && criterionValueString.equalsIgnoreCase(valueString)) + { + return (true); + } + } + } + } + } + if(value == null || !criterion.getValues().contains(value)) { return (false); @@ -456,6 +479,14 @@ public class BackendQueryFilterUtils value = String.valueOf(value); } + if(criterion.hasOption(CriteriaOption.CASE_INSENSITIVE)) + { + if(value instanceof String valueString && criteriaValue instanceof String criteriaValueString && valueString.equalsIgnoreCase(criteriaValueString)) + { + return (true); + } + } + if(!value.equals(criteriaValue)) { return (false); @@ -473,6 +504,14 @@ public class BackendQueryFilterUtils String stringValue = getStringFieldValue(value, fieldName, criterion); String criterionValue = getFirstStringCriterionValue(criterion); + if(criterion.hasOption(CriteriaOption.CASE_INSENSITIVE)) + { + if(stringValue.toLowerCase().contains(criterionValue.toLowerCase())) + { + return (true); + } + } + if(!stringValue.contains(criterionValue)) { return (false); @@ -491,6 +530,14 @@ public class BackendQueryFilterUtils String stringValue = getStringFieldValue(value, fieldName, criterion); String criterionValue = getFirstStringCriterionValue(criterion); + if(criterion.hasOption(CriteriaOption.CASE_INSENSITIVE)) + { + if(stringValue.toLowerCase().startsWith(criterionValue.toLowerCase())) + { + return (true); + } + } + if(!stringValue.startsWith(criterionValue)) { return (false); @@ -509,6 +556,14 @@ public class BackendQueryFilterUtils String stringValue = getStringFieldValue(value, fieldName, criterion); String criterionValue = getFirstStringCriterionValue(criterion); + if(criterion.hasOption(CriteriaOption.CASE_INSENSITIVE)) + { + if(stringValue.toLowerCase().endsWith(criterionValue.toLowerCase())) + { + return (true); + } + } + if(!stringValue.endsWith(criterionValue)) { return (false); @@ -665,4 +720,5 @@ public class BackendQueryFilterUtils regex.append("$"); return regex.toString(); } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index c29d1648..dabfb3f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -22,84 +22,108 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.InputStream; import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; -import com.kingsrook.qqq.backend.core.context.QContext; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; -import com.kingsrook.qqq.backend.core.state.AbstractStateKey; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; /******************************************************************************* ** Extract step for generic table bulk-insert ETL process + ** + ** This step does a little bit of transforming, actually - taking rows from + ** an uploaded file, and potentially merging them (for child-table use-cases) + ** and applying the "Mapping" - to put fully built records into the pipe for the + ** Transform step. *******************************************************************************/ public class BulkInsertExtractStep extends AbstractExtractStep { + + /*************************************************************************** + ** + ***************************************************************************/ @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME); - Optional optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey); - if(optionalUploadedFile.isEmpty()) + runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); + + int rowsAdded = 0; + int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); + + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface(); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) { - throw (new QException("Could not find uploaded file")); - } + /////////////////////////////////////////////////////////// + // read the header row (if this file & mapping uses one) // + /////////////////////////////////////////////////////////// + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; - byte[] bytes = optionalUploadedFile.get().getBytes(); - String fileName = optionalUploadedFile.get().getFilename(); + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under the limit - get more records form the file // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit) + { + int remainingLimit = originalLimit - rowsAdded; - ///////////////////////////////////////////////////// - // let the user specify field labels instead names // - ///////////////////////////////////////////////////// - QTableMetaData table = runBackendStepInput.getTable(); - String tableName = runBackendStepInput.getTableName(); - QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping(); - for(Map.Entry entry : table.getFields().entrySet()) - { - mapping.addMapping(entry.getKey(), entry.getValue().getLabel()); - } + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + int pageLimit = Math.min(remainingLimit, getMaxPageSize()); + List page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit); - ////////////////////////////////////////////////////////////////////////// - // get the non-editable fields - they'll be blanked out in a customizer // - ////////////////////////////////////////////////////////////////////////// - List nonEditableFields = table.getFields().values().stream() - .filter(f -> !f.getIsEditable()) - .toList(); - - if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) - { - new CsvToQRecordAdapter().buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() - .withRecordPipe(getRecordPipe()) - .withLimit(getLimit()) - .withCsv(new String(bytes)) - .withDoCorrectValueTypes(true) - .withTable(QContext.getQInstance().getTable(tableName)) - .withMapping(mapping) - .withRecordCustomizer((record) -> + if(page.size() > remainingLimit) { - //////////////////////////////////////////// - // remove values from non-editable fields // - //////////////////////////////////////////// - for(QFieldMetaData nonEditableField : nonEditableFields) - { - record.setValue(nonEditableField.getName(), null); - } - })); + ///////////////////////////////////////////////////////////// + // in case we got back more than we asked for, sub-list it // + ///////////////////////////////////////////////////////////// + page = page.subList(0, remainingLimit); + } + + ///////////////////////////////////////////// + // send this page of records into the pipe // + ///////////////////////////////////////////// + getRecordPipe().addRecords(page); + rowsAdded += page.size(); + } } - else + catch(QException qe) { - throw (new QUserFacingException("Unsupported file type.")); + throw qe; } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private int getMaxPageSize() + { + return (1000); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index c8b73ec2..30f98c91 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -22,16 +22,39 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* ** *******************************************************************************/ -public class BulkInsertLoadStep extends LoadViaInsertStep +public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSummaryProviderInterface { + private static final QLogger LOG = QLogger.getLogger(BulkInsertLoadStep.class); + + private Serializable firstInsertedPrimaryKey = null; + private Serializable lastInsertedPrimaryKey = null; + + /******************************************************************************* ** @@ -42,4 +65,66 @@ public class BulkInsertLoadStep extends LoadViaInsertStep return (QInputSource.USER); } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + super.runOnePage(runBackendStepInput, runBackendStepOutput); + + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName")); + + List insertedRecords = runBackendStepOutput.getRecords(); + for(QRecord insertedRecord : insertedRecords) + { + if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors())) + { + if(firstInsertedPrimaryKey == null) + { + firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + } + + lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + ArrayList processSummary = getTransformStep().getProcessSummary(runBackendStepOutput, isForResultScreen); + + try + { + if(firstInsertedPrimaryKey != null) + { + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepOutput.getValueString("tableName")); + QFieldMetaData field = table.getField(table.getPrimaryKeyField()); + if(field.getType().isNumeric()) + { + ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey); + if(Objects.equals(firstInsertedPrimaryKey, lastInsertedPrimaryKey)) + { + idsLine.setMessage("Inserted " + field.getLabel() + " " + firstInsertedPrimaryKey); + } + idsLine.setCount(null); + processSummary.add(idsLine); + } + } + } + catch(Exception e) + { + LOG.warn("Error adding inserted-keys process summary line", e); + } + + return (processSummary); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java new file mode 100644 index 00000000..c72df5fd --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -0,0 +1,175 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.File; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadMappingSuggester; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertPrepareFileMappingStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); + + String tableName = runBackendStepInput.getValueString("tableName"); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + runBackendStepOutput.addValue("tableStructure", tableStructure); + + boolean needSuggestedMapping = true; + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + needSuggestedMapping = false; + + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput, runBackendStepOutput); + } + + if(needSuggestedMapping) + { + @SuppressWarnings("unchecked") + List headerValues = (List) runBackendStepOutput.getValue("headerValues"); + buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void buildSuggestedMapping(List headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) + { + BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); + BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + runBackendStepOutput.addValue("suggestedBulkLoadProfile", bulkLoadProfile); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void buildFileDetailsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + File file = new File(storageInput.getReference()); + runBackendStepOutput.addValue("fileBaseName", file.getName()); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + ///////////////////////////////////////////////// + // read the 1st row, and assume it is a header // + ///////////////////////////////////////////////// + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + ArrayList headerValues = new ArrayList<>(); + ArrayList headerLetters = new ArrayList<>(); + for(int i = 0; i < headerRow.size(); i++) + { + headerValues.add(ValueUtils.getValueAsString(headerRow.getValue(i))); + headerLetters.add(toHeaderLetter(i)); + } + runBackendStepOutput.addValue("headerValues", headerValues); + runBackendStepOutput.addValue("headerLetters", headerLetters); + + /////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under preview-rows limit, read rows // + /////////////////////////////////////////////////////////////////////////////////////////// + int previewRows = 0; + int previewRowsLimit = 5; + ArrayList> bodyValues = new ArrayList<>(); + for(int i = 0; i < headerRow.size(); i++) + { + bodyValues.add(new ArrayList<>()); + } + + while(fileToRowsInterface.hasNext() && previewRows < previewRowsLimit) + { + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + previewRows++; + + for(int i = 0; i < headerRow.size(); i++) + { + bodyValues.get(i).add(ValueUtils.getValueAsString(bodyRow.getValueElseNull(i))); + } + } + runBackendStepOutput.addValue("bodyValuesPreview", bodyValues); + + } + catch(Exception e) + { + throw (new QException("Error reading bulk load file", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static String toHeaderLetter(int i) + { + StringBuilder rs = new StringBuilder(); + + do + { + rs.insert(0, (char) ('A' + (i % 26))); + i = (i / 26) - 1; + } + while(i >= 0); + + return (rs.toString()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java new file mode 100644 index 00000000..ea4810a9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -0,0 +1,302 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** step before the upload screen, to prepare dynamic help-text for user. + *******************************************************************************/ +public class BulkInsertPrepareFileUploadStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + runBackendStepOutput.addValue("theFile", null); + } + + String tableName = runBackendStepInput.getValueString("tableName"); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + runBackendStepOutput.addValue("tableStructure", tableStructure); + + List requiredFields = new ArrayList<>(); + List additionalFields = new ArrayList<>(); + for(QFieldMetaData field : tableStructure.getFields()) + { + if(field.getIsRequired()) + { + requiredFields.add(field); + } + else + { + additionalFields.add(field); + } + } + + StringBuilder html; + String childTableLabels = ""; + + StringBuilder tallCSV = new StringBuilder(); + StringBuilder wideCSV = new StringBuilder(); + StringBuilder flatCSV = new StringBuilder(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // potentially this could be a parameter - for now, hard-code false, but keep the code around that did this // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean listFieldsInHelpText = false; + + if(!CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) + { + html = new StringBuilder(""" +

Upload either a CSV or Excel (.xlsx) file, with one row for each record you want to + insert in the ${tableLabel} table.


+ +

Your file can contain any number of columns. You will be prompted to map fields from + the ${tableLabel} table to columns from your file or default values for all records that + you are loading on the next screen. It is optional whether you include a header row in your + file (though it is encouraged, and is the only way to received suggested field mappings). + For Excel files, only the first sheet in the workbook will be used.


+ """); + + if(listFieldsInHelpText) + { + appendTableRequiredAndAdditionalFields(html, requiredFields, additionalFields); + html.append(""" + Template: ${tableLabel}.csv"""); + } + else + { + html.append(""" +

You can download a template file to see the full list of available fields: + ${tableLabel}.csv +

+ """); + } + } + else + { + childTableLabels = StringUtils.joinWithCommasAndAnd(tableStructure.getAssociations().stream().map(a -> a.getLabel()).toList()) + " table" + StringUtils.plural(table.getAssociations()); + + html = new StringBuilder(""" +

Upload either a CSV or Excel (.xlsx) file. Your file can be in one of three layouts:

+ ${openUL} +

  • Flat: Each row in the file will create one record in the ${tableLabel} table.
  • +
  • Wide: Each row in the file will create one record in the ${tableLabel} table, + and optionally one or more records in the ${childTableLabels}, by supplying additional columns + for each sub-record that you want to create.
  • +
  • Tall: Rows with matching values in the fields being used for the ${tableLabel} + table will be used to create one ${tableLabel} record. One or more records will also be built + in the ${childTableLabels} by providing unique values in each row for the sub-records.
  • +
    + +

    Your file can contain any number of columns. You will be prompted to map fields from + the ${tableLabel} table to columns from your file or default values for all records that + you are loading on the next screen. It is optional whether you include a header row in your + file (though it is encouraged, and is the only way to received suggested field mappings). + For Excel files, only the first sheet in the workbook will be used.


    + """); + + if(listFieldsInHelpText) + { + appendTableRequiredAndAdditionalFields(html, requiredFields, additionalFields); + } + + addCsvFields(tallCSV, requiredFields, additionalFields); + addCsvFields(wideCSV, requiredFields, additionalFields); + + for(BulkLoadTableStructure association : tableStructure.getAssociations()) + { + if(listFieldsInHelpText) + { + html.append(""" +

    You can also add values for these ${childLabel} fields:

    + """.replace("${childLabel}", association.getLabel())); + appendFieldsAsUlToHtml(html, association.getFields()); + } + + addCsvFields(tallCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", ""); + addCsvFields(wideCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", " - 1"); + addCsvFields(wideCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", " - 2"); + } + + finishCSV(tallCSV); + finishCSV(wideCSV); + + if(listFieldsInHelpText) + { + html.append(""" + Templates: ${tableLabel} - Flat.csv + | ${tableLabel} - Tall.csv + | ${tableLabel} - Wide.csv + """); + } + else + { + html.append(""" +

    You can download a template file to see the full list of available fields: + ${tableLabel} - Flat.csv + | ${tableLabel} - Tall.csv + | ${tableLabel} - Wide.csv +

    + """); + } + } + + html.insert(0, """ +
    + File Upload Instructions +
    + """); + html.append("
    "); + + addCsvFields(flatCSV, requiredFields, additionalFields); + finishCSV(flatCSV); + + String htmlString = html.toString() + .replace("${tableLabel}", table.getLabel()) + .replace("${childTableLabels}", childTableLabels) + .replace("${flatCSV}", Base64.getEncoder().encodeToString(flatCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${tallCSV}", Base64.getEncoder().encodeToString(tallCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${wideCSV}", Base64.getEncoder().encodeToString(wideCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${openUL}", "
      "); + + runBackendStepOutput.addValue("upload.html", htmlString); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void finishCSV(StringBuilder flatCSV) + { + flatCSV.deleteCharAt(flatCSV.length() - 1); + flatCSV.append("\n"); + flatCSV.append(flatCSV.toString().replaceAll("[^,]", "")); + flatCSV.append("\n"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addCsvFields(StringBuilder csv, List requiredFields, List additionalFields) + { + addCsvFields(csv, requiredFields, additionalFields, "", ""); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addCsvFields(StringBuilder csv, List requiredFields, List additionalFields, String fieldLabelPrefix, String fieldLabelSuffix) + { + for(QFieldMetaData field : requiredFields) + { + csv.append(fieldLabelPrefix).append(field.getLabel()).append(fieldLabelSuffix).append(","); + } + + for(QFieldMetaData field : additionalFields) + { + csv.append(fieldLabelPrefix).append(field.getLabel()).append(fieldLabelSuffix).append(","); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void appendTableRequiredAndAdditionalFields(StringBuilder html, List requiredFields, List additionalFields) + { + if(!requiredFields.isEmpty()) + { + html.append(""" +

      You will be required to supply values (either in a column in the file, or by + choosing a default value on the next screen) for the following ${tableLabel} fields:

      + """); + appendFieldsAsUlToHtml(html, requiredFields); + } + + if(!additionalFields.isEmpty()) + { + if(requiredFields.isEmpty()) + { + html.append(""" +

      You can supply values (either in a column in the file, or by choosing a + default value on the next screen) for the following ${tableLabel} fields:

      + """); + } + else + { + html.append("

      You can also add values for these fields:

      "); + } + + appendFieldsAsUlToHtml(html, additionalFields); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void appendFieldsAsUlToHtml(StringBuilder html, List additionalFields) + { + html.append("${openUL}"); + for(QFieldMetaData field : additionalFields) + { + html.append("
    • ").append(field.getLabel()).append("
    • "); + } + html.append("

    "); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java new file mode 100644 index 00000000..8309531e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -0,0 +1,272 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.google.gson.reflect.TypeToken; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertPrepareValueMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertPrepareValueMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput, runBackendStepOutput); + } + + ///////////////////////////////////////////////////////////// + // prep the frontend for what field we're going to map now // + ///////////////////////////////////////////////////////////// + List fieldNamesToDoValueMapping = (List) runBackendStepInput.getValue("fieldNamesToDoValueMapping"); + Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex"); + if(valueMappingFieldIndex == null) + { + valueMappingFieldIndex = 0; + } + else + { + valueMappingFieldIndex++; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are no more fields (values) to map, then proceed to the standard streamed-ETL preview // + //////////////////////////////////////////////////////////////////////////////////////////////////// + if(valueMappingFieldIndex >= fieldNamesToDoValueMapping.size()) + { + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + return; + } + + runBackendStepInput.addValue("valueMappingFieldIndex", valueMappingFieldIndex); + + String fullFieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + String fieldNameWithoutWideSuffix = fullFieldName; + if(fieldNameWithoutWideSuffix.contains(",")) + { + fieldNameWithoutWideSuffix = fieldNameWithoutWideSuffix.replaceFirst(",.*", ""); + } + TableAndField tableAndField = getTableAndField(runBackendStepInput.getValueString("tableName"), fieldNameWithoutWideSuffix); + + runBackendStepInput.addValue("valueMappingField", new QFrontendFieldMetaData(tableAndField.field())); + runBackendStepInput.addValue("valueMappingFullFieldName", fullFieldName); + runBackendStepInput.addValue("valueMappingFieldTableName", tableAndField.table().getName()); + + //////////////////////////////////////////////////// + // get all the values from the file in this field // + // todo - should do all mapping fields at once? // + //////////////////////////////////////////////////// + ArrayList fileValues = getValuesForField(tableAndField.table(), tableAndField.field(), fullFieldName, runBackendStepInput); + runBackendStepOutput.addValue("fileValues", fileValues); + + /////////////////////////////////////////////// + // clear these in case not getting set below // + /////////////////////////////////////////////// + runBackendStepOutput.addValue("valueMapping", new HashMap<>()); + runBackendStepOutput.addValue("mappedValueLabels", new HashMap<>()); + + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + HashMap valueMapping = null; + if(bulkInsertMapping.getFieldNameToValueMapping() != null && bulkInsertMapping.getFieldNameToValueMapping().containsKey(fullFieldName)) + { + valueMapping = CollectionUtils.useOrWrap(bulkInsertMapping.getFieldNameToValueMapping().get(fullFieldName), new TypeToken<>() {}); + runBackendStepOutput.addValue("valueMapping", valueMapping); + + if(StringUtils.hasContent(tableAndField.field().getPossibleValueSourceName())) + { + HashMap possibleValueLabels = loadPossibleValues(tableAndField.field(), valueMapping); + runBackendStepOutput.addValue("mappedValueLabels", possibleValueLabels); + } + } + } + catch(Exception e) + { + LOG.warn("Error in bulk insert prepare value mapping", e); + throw new QException("Unhandled error in bulk insert prepare value mapping step", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static TableAndField getTableAndField(String tableName, String fullFieldName) throws QException + { + List parts = new ArrayList<>(List.of(fullFieldName.split("\\."))); + String fieldBaseName = parts.remove(parts.size() - 1); + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + for(String associationName : parts) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + table = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } + } + + TableAndField result = new TableAndField(table, table.getField(fieldBaseName)); + return result; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public record TableAndField(QTableMetaData table, QFieldMetaData field) {} + + + + /*************************************************************************** + ** + ***************************************************************************/ + private HashMap loadPossibleValues(QFieldMetaData field, Map valueMapping) throws QException + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); + input.setPossibleValueSourceName(field.getPossibleValueSourceName()); + input.setIdList(new ArrayList<>(new HashSet<>(valueMapping.values()))); // go through a set to strip dupes + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input); + + HashMap rs = new HashMap<>(); + for(QPossibleValue result : output.getResults()) + { + Serializable id = (Serializable) result.getId(); + rs.put(id, result.getLabel()); + } + return rs; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private ArrayList getValuesForField(QTableMetaData table, QFieldMetaData field, String fullFieldName, RunBackendStepInput runBackendStepInput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + + List wideAssociationIndexes = null; + if(fullFieldName.contains(",")) + { + wideAssociationIndexes = new ArrayList<>(); + String indexes = fullFieldName.substring(fullFieldName.lastIndexOf(",") + 1); + for(String index : indexes.split("\\.")) + { + wideAssociationIndexes.add(Integer.parseInt(index)); + } + } + + String associationNameChain = null; + if(fullFieldName.contains(".")) + { + associationNameChain = fullFieldName.substring(0, fullFieldName.lastIndexOf('.')); + } + + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + Set values = new LinkedHashSet<>(); + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; + Map fieldIndexes = bulkInsertMapping.getFieldIndexes(table, associationNameChain, headerRow, wideAssociationIndexes); + int index = fieldIndexes.get(field.getName()); + + while(fileToRowsInterface.hasNext()) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + Serializable value = row.getValueElseNull(index); + if(value != null) + { + values.add(ValueUtils.getValueAsString(value)); + } + + if(values.size() > 100) + { + throw (new QUserFacingException("Too many unique values were found for mapping for field: " + field.getName())); + } + } + + return (new ArrayList<>(values)); + } + catch(Exception e) + { + throw (new QException("Error getting values from file", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java new file mode 100644 index 00000000..94032fe5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -0,0 +1,250 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveFileMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveFileMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QRecord savedBulkLoadProfileRecord = BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + BulkLoadProfile bulkLoadProfile; + if(BulkInsertStepUtils.isHeadless(runBackendStepInput)) + { + ////////////////////////////////////////////////////////////////////////////// + // if running headless, build bulkLoadProfile from the saved profile record // + ////////////////////////////////////////////////////////////////////////////// + if(savedBulkLoadProfileRecord == null) + { + throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load")); + } + + SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord); + + try + { + bulkLoadProfile = JsonUtils.toObject(savedBulkLoadProfile.getMappingJson(), BulkLoadProfile.class); + } + catch(Exception e) + { + throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e)); + } + } + else + { + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + } + + ///////////////////////////////////////////////////////////////////////// + // put the list of bulk load profile into the process state - it's the // + // thing that the frontend will be looking at as the saved profile // + ///////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now build the mapping object that the backend wants - based on the bulkLoadProfile from the frontend // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); + bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); + bulkInsertMapping.setHasHeaderRow(bulkLoadProfile.getHasHeaderRow()); + bulkInsertMapping.setLayout(BulkInsertMapping.Layout.valueOf(bulkLoadProfile.getLayout())); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // handle field to name or index mappings (depending on if there's a header row being used) // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(BooleanUtils.isTrue(bulkInsertMapping.getHasHeaderRow())) + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + Map fieldNameToHeaderNameMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToHeaderNameMap(fieldNameToHeaderNameMap); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getHeaderName() != null) + { + String headerName = bulkLoadProfileField.getHeaderName(); + fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); + } + else if(bulkLoadProfileField.getColumnIndex() != null) + { + String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex())); + fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); + } + } + } + } + else + { + Map fieldNameToIndexMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToIndexMap(fieldNameToIndexMap); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getColumnIndex() != null) + { + fieldNameToIndexMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getColumnIndex()); + } + } + } + + ///////////////////////////////////// + // do fields w/ default values now // + ///////////////////////////////////// + HashMap fieldNameToDefaultValueMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToDefaultValueMap(fieldNameToDefaultValueMap); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getDefaultValue() != null) + { + fieldNameToDefaultValueMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getDefaultValue()); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // frontend at this point will have sent just told us which field names need value mapping // + // store those - and let them drive the value-mapping screens that we'll go through next // + // todo - uh, what if those come from profile, dummy!? + ///////////////////////////////////////////////////////////////////////////////////////////// + ArrayList fieldNamesToDoValueMapping = new ArrayList<>(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping())) + { + fieldNamesToDoValueMapping.add(bulkLoadProfileField.getFieldName()); + + if(CollectionUtils.nullSafeHasContents(bulkLoadProfileField.getValueMappings())) + { + bulkInsertMapping.getFieldNameToValueMapping().put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getValueMappings()); + } + } + } + runBackendStepOutput.addValue("fieldNamesToDoValueMapping", new ArrayList<>(fieldNamesToDoValueMapping)); + + /////////////////////////////////////////////////////////////////////////////////////// + // figure out what associations are being mapped, by looking at the full field names // + /////////////////////////////////////////////////////////////////////////////////////// + Set associationNameSet = new HashSet<>(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getFieldName().contains(".")) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // handle parent.child.grandchild.fieldName,index.index.index if we do sub-indexes for grandchildren... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNameBeforeIndex = bulkLoadProfileField.getFieldName().split(",")[0]; + associationNameSet.add(fieldNameBeforeIndex.substring(0, fieldNameBeforeIndex.lastIndexOf('.'))); + } + } + bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet)); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // at this point we're done populating the bulkInsertMapping object. put it in the process state. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + if(BulkInsertStepUtils.isHeadless(runBackendStepInput)) + { + //////////////////////////////////////////////////////////////////////// + // if running headless, always go straight to the preview screen next // + // todo actually, we could make this execute, right? // + //////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + } + else + { + if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + { + ////////////////////////////////////////////////////////////////////////////////// + // just go to the prepareValueMapping backend step - it'll figure out the rest. // + // it's also where the value-mapping loop of steps points. // + // and, this will actually be the default (e.g., the step after this one). // + ////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.addValue("valueMappingFieldIndex", -1); + } + else + { + ////////////////////////////////////////////////////////////////////////////////// + // else - if no values to map - continue with the standard streamed-ETL preview // + ////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + } + } + } + catch(Exception e) + { + LOG.warn("Error in bulk insert receive mapping", e); + throw new QException("Unhandled error in bulk insert receive mapping step", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java new file mode 100644 index 00000000..e1b5bc9a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.core.type.TypeReference; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveValueMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveValueMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + List fieldNamesToDoValueMapping = (List) runBackendStepInput.getValue("fieldNamesToDoValueMapping"); + Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex"); + + String fieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////// + // put the list of bulk load profile into the process state - it's the // + // thing that the frontend will be looking at as the saved profile // + ///////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // get the bulkInsertMapping object from the process, creating a fieldNameToValueMapping map within it if needed // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + Map> fieldNameToValueMapping = bulkInsertMapping.getFieldNameToValueMapping(); + if(fieldNameToValueMapping == null) + { + fieldNameToValueMapping = new HashMap<>(); + bulkInsertMapping.setFieldNameToValueMapping(fieldNameToValueMapping); + } + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + //////////////////////////////////////////////// + // put the mapped values into the mapping map // + //////////////////////////////////////////////// + Map mappedValues = JsonUtils.toObject(runBackendStepInput.getValueString("mappedValuesJSON"), new TypeReference<>() {}); + fieldNameToValueMapping.put(fieldName, mappedValues); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // always return to the prepare-mapping step - as it will determine if it's time to break the loop or not. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepPrepareValueMapping(runBackendStepOutput); + } + catch(Exception e) + { + LOG.warn("Error in bulk insert receive mapping", e); + throw new QException("Unhandled error in bulk insert receive mapping step", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java new file mode 100644 index 00000000..e9c09cea --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -0,0 +1,219 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerKeyRecordMessage; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.json.JSONArray; +import org.json.JSONObject; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertStepUtils +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public static StorageInput getStorageInputForTheFile(RunBackendStepInput input) throws QException + { + @SuppressWarnings("unchecked") + ArrayList storageInputs = (ArrayList) input.getValue("theFile"); + if(storageInputs == null) + { + throw (new QException("StorageInputs for theFile were not found in process state")); + } + + if(storageInputs.isEmpty()) + { + throw (new QException("StorageInputs for theFile was an empty list")); + } + + StorageInput storageInput = storageInputs.get(0); + return (storageInput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setStorageInputForTheFile(RunProcessInput runProcessInput, StorageInput storageInput) + { + ArrayList storageInputs = new ArrayList<>(); + storageInputs.add(storageInput); + runProcessInput.addValue("theFile", storageInputs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setNextStepStreamedETLPreview(RunBackendStepOutput runBackendStepOutput) + { + runBackendStepOutput.setOverrideLastStepName("receiveValueMapping"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setNextStepPrepareValueMapping(RunBackendStepOutput runBackendStepOutput) + { + runBackendStepOutput.setOverrideLastStepName("receiveFileMapping"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static BulkLoadProfile getBulkLoadProfile(RunBackendStepInput runBackendStepInput) + { + String version = runBackendStepInput.getValueString("version"); + if("v1".equals(version)) + { + String layout = runBackendStepInput.getValueString("layout"); + Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow"); + + ArrayList fieldList = new ArrayList<>(); + + JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON")); + for(int i = 0; i < array.length(); i++) + { + JSONObject jsonObject = array.getJSONObject(i); + BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); + fieldList.add(bulkLoadProfileField); + bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName")); + bulkLoadProfileField.setHeaderName(jsonObject.has("headerName") ? jsonObject.getString("headerName") : null); + bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); + bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); + bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); + + if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings")) + { + bulkLoadProfileField.setValueMappings(new HashMap<>()); + JSONObject valueMappingsJsonObject = jsonObject.getJSONObject("valueMappings"); + for(String fileValue : valueMappingsJsonObject.keySet()) + { + bulkLoadProfileField.getValueMappings().put(fileValue, ValueUtils.getValueAsString(valueMappingsJsonObject.get(fileValue))); + } + } + } + + BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withVersion(version) + .withFieldList(fieldList) + .withHasHeaderRow(hasHeaderRow) + .withLayout(layout); + + return (bulkLoadProfile); + } + else + { + throw (new IllegalArgumentException("Unexpected version for bulk load profile: " + version)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QRecord handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId"); + if(savedBulkLoadProfileId != null) + { + QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId); + runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord); + return (savedBulkLoadProfileRecord); + } + + return (null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static boolean isHeadless(RunBackendStepInput runBackendStepInput) + { + return (runBackendStepInput.getValuePrimitiveBoolean("isHeadless")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setHeadless(RunProcessInput runProcessInput) + { + runProcessInput.addValue("isHeadless", true); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setProcessTracerKeyRecordMessage(RunProcessInput runProcessInput, ProcessTracerKeyRecordMessage processTracerKeyRecordMessage) + { + runProcessInput.addValue("processTracerKeyRecordMessage", processTracerKeyRecordMessage); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProcessTracerKeyRecordMessage getProcessTracerKeyRecordMessage(RunBackendStepInput runBackendStepInput) + { + Serializable value = runBackendStepInput.getValue("processTracerKeyRecordMessage"); + if(value instanceof ProcessTracerKeyRecordMessage processTracerKeyRecordMessage) + { + return (processTracerKeyRecordMessage); + } + + return (null); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index f4eff9a3..410fccf2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; @@ -47,15 +48,26 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.AbstractBulkLoadRollableValueError; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -65,9 +77,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep { private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); - private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") + .withDoReplaceSingletonCountLinesWithSuffixOnly(false); - private Map ukErrorSummaries = new HashMap<>(); + private ListingHash errorToExampleRowValueMap = new ListingHash<>(); + private ListingHash errorToExampleRowsMap = new ListingHash<>(); + + private Map ukErrorSummaries = new HashMap<>(); + private Map associationsToInsertSummaries = new HashMap<>(); private QTableMetaData table; @@ -75,6 +92,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep private int rowsProcessed = 0; + private static final int EXAMPLE_ROW_LIMIT = 10; + /******************************************************************************* @@ -111,6 +130,53 @@ public class BulkInsertTransformStep extends AbstractTransformStep // since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that if a saved profile was selected on a review screen, that the result screen knows about it. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up the validationReview widget to render preview records using the table layout, and including the associations // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("formatPreviewRecordUsingTableLayout", table.getName()); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(table.getName()); + if(CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) + { + ArrayList previewRecordAssociatedTableNames = new ArrayList<>(); + ArrayList previewRecordAssociatedWidgetNames = new ArrayList<>(); + ArrayList previewRecordAssociationNames = new ArrayList<>(); + + //////////////////////////////////////////////////////////// + // note - not recursively processing associations here... // + //////////////////////////////////////////////////////////// + for(BulkLoadTableStructure associatedStructure : tableStructure.getAssociations()) + { + String associationName = associatedStructure.getAssociationPath(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + for(QFieldSection section : table.getSections()) + { + QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(section.getWidgetName()); + if(widget != null && WidgetType.CHILD_RECORD_LIST.getType().equals(widget.getType())) + { + Serializable widgetJoinName = widget.getDefaultValues().get("joinName"); + if(Objects.equals(widgetJoinName, association.get().getJoinName())) + { + previewRecordAssociatedTableNames.add(association.get().getAssociatedTableName()); + previewRecordAssociatedWidgetNames.add(widget.getName()); + previewRecordAssociationNames.add(association.get().getName()); + } + } + } + } + } + runBackendStepOutput.addValue("previewRecordAssociatedTableNames", previewRecordAssociatedTableNames); + runBackendStepOutput.addValue("previewRecordAssociatedWidgetNames", previewRecordAssociatedWidgetNames); + runBackendStepOutput.addValue("previewRecordAssociationNames", previewRecordAssociationNames); + } } @@ -121,8 +187,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - int rowsInThisPage = runBackendStepInput.getRecords().size(); - QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + List records = runBackendStepInput.getRecords(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // @@ -130,7 +196,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep InsertInput insertInput = new InsertInput(); insertInput.setInputSource(QInputSource.USER); insertInput.setTableName(runBackendStepInput.getTableName()); - insertInput.setRecords(runBackendStepInput.getRecords()); + insertInput.setRecords(records); insertInput.setSkipUniqueKeyCheck(true); ////////////////////////////////////////////////////////////////////// @@ -139,27 +205,35 @@ public class BulkInsertTransformStep extends AbstractTransformStep // we do this, in case it needs to, for example, adjust values that // // are part of a unique key // ////////////////////////////////////////////////////////////////////// - Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + boolean didAlreadyRunCustomizer = false; + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { - List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true); + List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, records, true); runBackendStepInput.setRecords(recordsAfterCustomizer); - /////////////////////////////////////////////////////////////////////////////////////// - // todo - do we care if the customizer runs both now, and in the validation below? // - // right now we'll let it run both times, but maybe that should be protected against // - /////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // so we used to have a comment here asking "do we care if the customizer runs both now, and in the validation below?" // + // when implementing Bulk Load V2, we were seeing that some customizers were adding errors to records, both now, and // + // when they ran below. so, at that time, we added this boolean, to track and avoid the double-run... // + // we could also imagine this being a setting on the pre-insert customizer, similar to its whenToRun attribute... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + didAlreadyRunCustomizer = true; } } + /////////////////////////////////////////////////////////////////////////////// + // If the table has unique keys - then capture all values on these records // + // for each key and set up a processSummaryLine for each of the table's UK's // + /////////////////////////////////////////////////////////////////////////////// Map>> existingKeys = new HashMap<>(); List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); for(UniqueKey uniqueKey : uniqueKeys) { - existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet()); + existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, records, uniqueKey).keySet()); ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR)); } @@ -187,14 +261,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction // // will only be getting the records in pages, but in here, we'll track UK's across pages!! // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table); + List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(records, existingKeys, uniqueKeys, table); ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// insertInput.setRecords(recordsWithoutUkErrors); InsertAction insertAction = new InsertAction(); - insertAction.performValidations(insertInput, true); + insertAction.performValidations(insertInput, true, didAlreadyRunCustomizer); List validationResultRecords = insertInput.getRecords(); ///////////////////////////////////////////////////////////////// @@ -203,10 +277,29 @@ public class BulkInsertTransformStep extends AbstractTransformStep List outputRecords = new ArrayList<>(); for(QRecord record : validationResultRecords) { + List errorsFromAssociations = getErrorsFromAssociations(record); + if(CollectionUtils.nullSafeHasContents(errorsFromAssociations)) + { + List recordErrors = Objects.requireNonNullElseGet(record.getErrors(), () -> new ArrayList<>()); + recordErrors.addAll(errorsFromAssociations); + record.setErrors(recordErrors); + } + if(CollectionUtils.nullSafeHasContents(record.getErrors())) { - String message = record.getErrors().get(0).getMessage(); - processSummaryWarningsAndErrorsRollup.addError(message, null); + for(QErrorMessage error : record.getErrors()) + { + if(error instanceof AbstractBulkLoadRollableValueError rollableValueError) + { + processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(rollableValueError, record); + } + else + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + addToErrorToExampleRowMap(error.getMessage(), record); + } + } } else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) { @@ -218,12 +311,77 @@ public class BulkInsertTransformStep extends AbstractTransformStep { okSummary.incrementCountAndAddPrimaryKey(null); outputRecords.add(record); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + String associationName = entry.getKey(); + ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK)); + associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size()); + } } } runBackendStepOutput.setRecords(outputRecords); + this.rowsProcessed += records.size(); + } - this.rowsProcessed += rowsInThisPage; + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getErrorsFromAssociations(QRecord record) + { + List rs = null; + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + for(QRecord associatedRecord : CollectionUtils.nonNullList(entry.getValue())) + { + if(CollectionUtils.nullSafeHasContents(associatedRecord.getErrors())) + { + rs = Objects.requireNonNullElseGet(rs, () -> new ArrayList<>()); + rs.addAll(associatedRecord.getErrors()); + + List childErrors = getErrorsFromAssociations(associatedRecord); + if(CollectionUtils.nullSafeHasContents(childErrors)) + { + rs.addAll(childErrors); + } + } + } + } + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addToErrorToExampleRowValueMap(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record) + { + String message = bulkLoadRollableValueError.getMessageToUseAsProcessSummaryRollupKey(); + List rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>()); + + if(rowValues.size() < EXAMPLE_ROW_LIMIT) + { + rowValues.add(new RowValue(bulkLoadRollableValueError, record)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addToErrorToExampleRowMap(String message, QRecord record) + { + List rowNos = errorToExampleRowsMap.computeIfAbsent(message, k -> new ArrayList<>()); + + if(rowNos.size() < EXAMPLE_ROW_LIMIT) + { + rowNos.add(BulkLoadRecordUtils.getRowNosString(record)); + } } @@ -231,7 +389,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep /******************************************************************************* ** *******************************************************************************/ - private List getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map>> existingKeys, List uniqueKeys, QTableMetaData table) + private List getRecordsWithoutUniqueKeyErrors(List records, Map>> existingKeys, List uniqueKeys, QTableMetaData table) { //////////////////////////////////////////////////// // if there are no UK's, proceed with all records // @@ -239,7 +397,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep List recordsWithoutUkErrors = new ArrayList<>(); if(existingKeys.isEmpty()) { - recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); + recordsWithoutUkErrors.addAll(records); } else { @@ -255,7 +413,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep // else, get each records keys and see if it already exists or not // // also, build a set of keys we've seen (within this page (or overall?)) // /////////////////////////////////////////////////////////////////////////// - for(QRecord record : runBackendStepInput.getRecords()) + for(QRecord record : records) { if(CollectionUtils.nullSafeHasContents(record.getErrors())) { @@ -318,6 +476,15 @@ public class BulkInsertTransformStep extends AbstractTransformStep ArrayList rs = new ArrayList<>(); String tableLabel = table == null ? "" : table.getLabel(); + ProcessSummaryLine recordsProcessedLine = new ProcessSummaryLine(Status.INFO); + recordsProcessedLine.setCount(rowsProcessed); + rs.add(recordsProcessedLine); + recordsProcessedLine.withMessageSuffix(" processed from the file."); + recordsProcessedLine.withSingularFutureMessage("record was"); + recordsProcessedLine.withSingularPastMessage("record was"); + recordsProcessedLine.withPluralFutureMessage("records were"); + recordsProcessedLine.withPluralPastMessage("records were"); + String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; okSummary.setSingularFutureMessage(tableLabel + " record will be inserted" + noWarningsSuffix + "."); okSummary.setPluralFutureMessage(tableLabel + " records will be inserted" + noWarningsSuffix + "."); @@ -326,6 +493,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); + for(Map.Entry entry : associationsToInsertSummaries.entrySet()) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(entry.getKey())).findFirst(); + if(association.isPresent()) + { + QTableMetaData associationTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + String associationLabel = associationTable.getLabel(); + + ProcessSummaryLine line = entry.getValue(); + line.setSingularFutureMessage(associationLabel + " record will be inserted."); + line.setPluralFutureMessage(associationLabel + " records will be inserted."); + line.setSingularPastMessage(associationLabel + " record was inserted."); + line.setPluralPastMessage(associationLabel + " records were inserted."); + line.pickMessage(isForResultScreen); + line.addSelfToListIfAnyCount(rs); + } + } + for(Map.Entry entry : ukErrorSummaries.entrySet()) { UniqueKey uniqueKey = entry.getKey(); @@ -333,8 +518,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" - + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") - + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) + + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) .withSingularFutureMessage(" record will not be") .withPluralFutureMessage(" records will not be") @@ -344,9 +529,76 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary.addSelfToListIfAnyCount(rs); } + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for process summary lines that exist in the error-to-example-row-value map, add those example values to the lines. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(Map.Entry entry : processSummaryWarningsAndErrorsRollup.getErrorSummaries().entrySet()) + { + String message = entry.getKey(); + if(errorToExampleRowValueMap.containsKey(message)) + { + ProcessSummaryLine line = entry.getValue(); + List rowValues = errorToExampleRowValueMap.get(message); + String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; + line.setMessageSuffix(line.getMessageSuffix() + periodIfNeeded(line.getMessageSuffix()) + " " + exampleOrFull + "Values:"); + line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList())); + } + else if(errorToExampleRowsMap.containsKey(message)) + { + ProcessSummaryLine line = entry.getValue(); + List rowDescriptions = errorToExampleRowsMap.get(message); + String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; + line.setMessageSuffix(line.getMessageSuffix() + periodIfNeeded(line.getMessageSuffix()) + " " + exampleOrFull + "Records:"); + line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList())); + } + } + processSummaryWarningsAndErrorsRollup.addToList(rs); return (rs); } + + + /*************************************************************************** + * + ***************************************************************************/ + private String periodIfNeeded(String input) + { + if(input != null && input.matches(".*\\. *$")) + { + return (""); + } + + return ("."); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private record RowValue(String row, String value) + { + + /*************************************************************************** + ** + ***************************************************************************/ + public RowValue(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record) + { + this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadRollableValueError.getValue())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + return row + " [" + value + "]"; + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java new file mode 100644 index 00000000..88437f1b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -0,0 +1,139 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.util.Iterator; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private Iterator iterator; + + private boolean useLast = false; + private BulkLoadFileRow last; + + int rowNo = 0; + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean hasNext() + { + if(iterator == null) + { + throw new IllegalStateException("Object was not init'ed"); + } + + if(useLast) + { + return true; + } + + return iterator.hasNext(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow next() + { + rowNo++; + if(iterator == null) + { + throw new IllegalStateException("Object was not init'ed"); + } + + if(useLast) + { + useLast = false; + return (this.last); + } + + E e = iterator.next(); + + BulkLoadFileRow row = makeRow(e); + + this.last = row; + return (this.last); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract BulkLoadFileRow makeRow(E e); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void unNext() + { + rowNo--; + useLast = true; + } + + + + /******************************************************************************* + ** Getter for iterator + *******************************************************************************/ + public Iterator getIterator() + { + return (this.iterator); + } + + + + /******************************************************************************* + ** Setter for iterator + *******************************************************************************/ + public void setIterator(Iterator iterator) + { + this.iterator = iterator; + } + + + + /******************************************************************************* + ** Getter for rowNo + ** + *******************************************************************************/ + @Override + public int getRowNo() + { + return rowNo; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java new file mode 100644 index 00000000..332c8722 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class CsvFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private CSVParser csvParser; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static CsvFileToRows forString(String csv) throws QException + { + CsvFileToRows csvFileToRows = new CsvFileToRows(); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(csv.getBytes()); + csvFileToRows.init(byteArrayInputStream); + + return (csvFileToRows); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + try + { + csvParser = new CSVParser(new InputStreamReader(inputStream), CSVFormat.DEFAULT + .withIgnoreSurroundingSpaces() + ); + setIterator(csvParser.iterator()); + } + catch(IOException e) + { + throw new QException("Error opening CSV Parser", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(CSVRecord csvRecord) + { + Serializable[] values = new Serializable[csvRecord.size()]; + int i = 0; + for(String s : csvRecord) + { + values[i++] = s; + } + + return (new BulkLoadFileRow(values, getRowNo())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + if(csvParser != null) + { + csvParser.close(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java new file mode 100644 index 00000000..9ab6cce3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.InputStream; +import java.util.Iterator; +import java.util.Locale; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface FileToRowsInterface extends AutoCloseable, Iterator +{ + + /*************************************************************************** + ** + ***************************************************************************/ + static FileToRowsInterface forFile(String fileName, InputStream inputStream) throws QException + { + FileToRowsInterface rs; + if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) + { + rs = new CsvFileToRows(); + } + else if(fileName.toLowerCase(Locale.ROOT).endsWith(".xlsx")) + { + rs = new XlsxFileToRows(); + } + else + { + throw (new QUserFacingException("Unrecognized file extension - expecting .csv or .xlsx")); + } + + rs.init(inputStream); + return rs; + } + + + /******************************************************************************* + ** + *******************************************************************************/ + void init(InputStream inputStream) throws QException; + + + /*************************************************************************** + ** + ***************************************************************************/ + int getRowNo(); + + + /*************************************************************************** + ** + ***************************************************************************/ + void unNext(); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java new file mode 100644 index 00000000..289b90ee --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -0,0 +1,263 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.dhatim.fastexcel.reader.Cell; +import org.dhatim.fastexcel.reader.ReadableWorkbook; +import org.dhatim.fastexcel.reader.ReadingOptions; +import org.dhatim.fastexcel.reader.Row; +import org.dhatim.fastexcel.reader.Sheet; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class XlsxFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private static final QLogger LOG = QLogger.getLogger(XlsxFileToRows.class); + + private static final Pattern DAY_PATTERN = Pattern.compile(".*\\b(d|dd)\\b.*"); + + private ReadableWorkbook workbook; + private Stream rows; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + try + { + workbook = new ReadableWorkbook(inputStream, new ReadingOptions(true, true)); + Sheet sheet = workbook.getFirstSheet(); + + rows = sheet.openStream(); + setIterator(rows.iterator()); + } + catch(IOException e) + { + throw new QException("Error opening XLSX Parser", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(org.dhatim.fastexcel.reader.Row readerRow) + { + Serializable[] values = new Serializable[readerRow.getCellCount()]; + + for(int i = 0; i < readerRow.getCellCount(); i++) + { + values[i] = processCell(readerRow, i); + } + + return new BulkLoadFileRow(values, getRowNo()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private Serializable processCell(Row readerRow, int columnIndex) + { + Cell cell = readerRow.getCell(columnIndex); + if(cell == null) + { + return (null); + } + + String dataFormatString = cell.getDataFormatString(); + switch(cell.getType()) + { + case NUMBER -> + { + ///////////////////////////////////////////////////////////////////////////////////// + // dates, date-times, integers, and decimals are all identified as type = "number" // + // so go through this process to try to identify what user means it as // + ///////////////////////////////////////////////////////////////////////////////////// + if(isDateTimeFormat(dataFormatString)) + { + //////////////////////////////////////////////////////////////////////////////////////// + // first - if it has a date-time looking format string, then treat it as a date-time. // + //////////////////////////////////////////////////////////////////////////////////////// + return (cell.asDate()); + } + else if(isDateFormat(dataFormatString)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // second, if it has a date looking format string (which is a sub-set of date-time), then treat as date. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (cell.asDate().toLocalDate()); + } + else + { + //////////////////////////////////////////////////////////////////////////////////////// + // now assume it's a number - but in case this optional is empty (why?) return a null // + //////////////////////////////////////////////////////////////////////////////////////// + Optional bigDecimal = readerRow.getCellAsNumber(columnIndex); + if(bigDecimal.isEmpty()) + { + return (null); + } + + try + { + //////////////////////////////////////////////////////////// + // now if the bigDecimal is an exact integer, return that // + //////////////////////////////////////////////////////////// + Integer i = bigDecimal.get().intValueExact(); + return (i); + } + catch(ArithmeticException e) + { + ///////////////////////////////// + // else, end up with a decimal // + ///////////////////////////////// + return (bigDecimal.get()); + } + } + } + case STRING -> + { + return cell.asString(); + } + case BOOLEAN -> + { + return cell.asBoolean(); + } + case FORMULA -> + { + return (ValueUtils.getValueAsString(cell.getRawValue())); + } + case EMPTY, ERROR -> + { + LOG.debug("Empty or Error cell", logPair("type", cell.getType()), logPair("rawValue", () -> cell.getRawValue())); + return (null); + } + default -> + { + return (null); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean isDateTimeFormat(String dataFormatString) + { + if(dataFormatString == null) + { + return (false); + } + + if(hasDay(dataFormatString) && hasHour(dataFormatString)) + { + return (true); + } + + return false; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean hasHour(String dataFormatString) + { + return dataFormatString.contains("h") || dataFormatString.contains("H"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean hasDay(String dataFormatString) + { + return DAY_PATTERN.matcher(dataFormatString).matches(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean isDateFormat(String dataFormatString) + { + if(dataFormatString == null) + { + return (false); + } + + if(hasDay(dataFormatString)) + { + return (true); + } + + return false; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + if(workbook != null) + { + workbook.close(); + } + + if(rows != null) + { + rows.close(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java new file mode 100644 index 00000000..74082fd7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractBulkLoadRollableValueError extends BadInputStatusMessage +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public AbstractBulkLoadRollableValueError(String message) + { + super(message); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract String getMessageToUseAsProcessSummaryRollupKey(); + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract Serializable getValue(); +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java new file mode 100644 index 00000000..71f0edcc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java @@ -0,0 +1,232 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; + + +/******************************************************************************* + ** Given a bulk-upload, create a suggested mapping + *******************************************************************************/ +public class BulkLoadMappingSuggester +{ + private Map massagedHeadersWithoutNumbersToIndexMap; + private Map massagedHeadersWithNumbersToIndexMap; + + private String layout = "FLAT"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List headerRow) + { + massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = massageHeader(headerRow.get(i), true); + + if(!massagedHeadersWithoutNumbersToIndexMap.containsKey(headerValue)) + { + massagedHeadersWithoutNumbersToIndexMap.put(headerValue, i); + } + } + + massagedHeadersWithNumbersToIndexMap = new LinkedHashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = massageHeader(headerRow.get(i), false); + + if(!massagedHeadersWithNumbersToIndexMap.containsKey(headerValue)) + { + massagedHeadersWithNumbersToIndexMap.put(headerValue, i); + } + } + + ArrayList fieldList = new ArrayList<>(); + processTable(tableStructure, fieldList, headerRow); + + ///////////////////////////////////////////////// + // sort the fields to match the column indexes // + ///////////////////////////////////////////////// + fieldList.sort(Comparator.comparing(blpf -> blpf.getColumnIndex())); + + BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withVersion("v1") + .withLayout(layout) + .withHasHeaderRow(true) + .withFieldList(fieldList); + + return (bulkLoadProfile); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processTable(BulkLoadTableStructure tableStructure, ArrayList fieldList, List headerRow) + { + Map rs = new HashMap<>(); + for(QFieldMetaData field : tableStructure.getFields()) + { + String fieldName = massageHeader(field.getName(), false); + String fieldLabel = massageHeader(field.getLabel(), false); + String tablePlusFieldLabel = massageHeader(QContext.getQInstance().getTable(tableStructure.getTableName()).getLabel() + ": " + field.getLabel(), false); + String fullFieldName = (StringUtils.hasContent(tableStructure.getAssociationPath()) ? (tableStructure.getAssociationPath() + ".") : "") + field.getName(); + + //////////////////////////////////////////////////////////////////////////////////// + // consider, if this is a many-table, if there are many matches, for wide mode... // + //////////////////////////////////////////////////////////////////////////////////// + if(tableStructure.getIsMany()) + { + List matchingIndexes = new ArrayList<>(); + + for(Map.Entry entry : massagedHeadersWithNumbersToIndexMap.entrySet()) + { + String header = entry.getKey(); + if(header.matches(fieldName + "\\d*$") || header.matches(fieldLabel + "\\d*$")) + { + matchingIndexes.add(entry.getValue()); + } + } + + if(CollectionUtils.nullSafeHasContents(matchingIndexes)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we found more than 1 match - consider this a likely wide file, and build fields as wide-fields // + // else, if only 1, allow us to go down into the TALL block below // + // note - should we do a merger at the end, in case we found some wide, some tall? // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + if(matchingIndexes.size() > 1) + { + layout = "WIDE"; + + int i = 0; + for(Integer index : matchingIndexes) + { + fieldList.add(new BulkLoadProfileField() + .withFieldName(fullFieldName + "," + i) + .withHeaderName(headerRow.get(index)) + .withColumnIndex(index) + ); + + i++; + } + + continue; + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - look for matches, first w/ headers with numbers, then headers w/o numbers checking labels and names // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Integer index = null; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for each of these potential identities of the field: // + // 1) its label, massaged // + // 2) its name, massaged // + // 3) its label, massaged, with numbers stripped away // + // 4) its name, massaged, with numbers stripped away // + // check if that identity is in the massagedHeadersWithNumbersToIndexMap, or the massagedHeadersWithoutNumbersToIndexMap. // + // this is currently successful in the both versions of the address 1 / address 2 <=> address / address 2 use-case // + // that is, BulkLoadMappingSuggesterTest.testChallengingAddress1And2 // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(String fieldIdentity : ListBuilder.of(fieldLabel, fieldName, tablePlusFieldLabel, massageHeader(fieldLabel, true), massageHeader(fieldName, true))) + { + if(massagedHeadersWithNumbersToIndexMap.containsKey(fieldIdentity)) + { + index = massagedHeadersWithNumbersToIndexMap.get(fieldIdentity); + } + else if(massagedHeadersWithoutNumbersToIndexMap.containsKey(fieldIdentity)) + { + index = massagedHeadersWithoutNumbersToIndexMap.get(fieldIdentity); + } + + if(index != null) + { + break; + } + } + + if(index != null) + { + fieldList.add(new BulkLoadProfileField() + .withFieldName(fullFieldName) + .withHeaderName(headerRow.get(index)) + .withColumnIndex(index) + ); + + if(tableStructure.getIsMany() && layout.equals("FLAT")) + { + ////////////////////////////////////////////////////////////////////////////////////////// + // the first time we find an is-many child, if we were still marked as flat, go to tall // + ////////////////////////////////////////////////////////////////////////////////////////// + layout = "TALL"; + } + } + } + + //////////////////////////////////////////// + // recursively process child associations // + //////////////////////////////////////////// + for(BulkLoadTableStructure associationTableStructure : CollectionUtils.nonNullList(tableStructure.getAssociations())) + { + processTable(associationTableStructure, fieldList, headerRow); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String massageHeader(String header, boolean stripNumbers) + { + if(header == null) + { + return (null); + } + + String massagedWithNumbers = header.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", ""); + return stripNumbers ? massagedWithNumbers.replaceAll("[0-9]", "") : massagedWithNumbers; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java new file mode 100644 index 00000000..199d775f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Specialized error for records, for bulk-load use-cases, where we want to + ** report back info to the user about the field & value. + *******************************************************************************/ +public class BulkLoadPossibleValueError extends AbstractBulkLoadRollableValueError +{ + private final String fieldLabel; + private final Serializable value; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadPossibleValueError(String fieldName, Serializable value, String fieldLabel) + { + super("Value [" + value + "] for field [" + fieldLabel + "] is not a valid option"); + this.value = value; + this.fieldLabel = fieldLabel; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getMessageToUseAsProcessSummaryRollupKey() + { + return ("Unrecognized value for field [" + fieldLabel + "]"); + } + + + + /******************************************************************************* + ** Getter for value + ** + *******************************************************************************/ + @Override + public Serializable getValue() + { + return value; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java new file mode 100644 index 00000000..0c700fe0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Utility methods for working with records in a bulk-load. + ** + ** Originally added for working with backendDetails around the source rows. + *******************************************************************************/ +public class BulkLoadRecordUtils +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public static QRecord addBackendDetailsAboutFileRows(QRecord record, BulkLoadFileRow fileRow) + { + return (addBackendDetailsAboutFileRows(record, new ArrayList<>(List.of(fileRow)))); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public static QRecord addBackendDetailsAboutFileRows(QRecord record, ArrayList fileRows) + { + if(CollectionUtils.nullSafeHasContents(fileRows)) + { + Integer firstRowNo = fileRows.get(0).getRowNo(); + Integer lastRowNo = fileRows.get(fileRows.size() - 1).getRowNo(); + + if(Objects.equals(firstRowNo, lastRowNo)) + { + record.addBackendDetail("rowNos", "Row " + firstRowNo); + } + else + { + record.addBackendDetail("rowNos", "Rows " + firstRowNo + "-" + lastRowNo); + } + } + else + { + record.addBackendDetail("rowNos", "Rows ?"); + } + + record.addBackendDetail("fileRows", fileRows); + return (record); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static String getRowNosString(QRecord record) + { + return (record.getBackendDetailString("rowNos")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @SuppressWarnings("unchecked") + public static ArrayList getFileRows(QRecord record) + { + return (ArrayList) record.getBackendDetail("fileRows"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static List getFileRowNos(QRecord record) + { + return (getFileRows(record).stream().map(row -> row.getRowNo()).toList()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java new file mode 100644 index 00000000..26575be3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java @@ -0,0 +1,142 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** utility to build BulkLoadTableStructure objects for a QQQ Table. + *******************************************************************************/ +public class BulkLoadTableStructureBuilder +{ + /*************************************************************************** + ** + ***************************************************************************/ + public static BulkLoadTableStructure buildTableStructure(String tableName) + { + return (buildTableStructure(tableName, null, null)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); + tableStructure.setTableName(tableName); + tableStructure.setLabel(table.getLabel()); + + Set associationJoinFieldNamesToExclude = new HashSet<>(); + + if(association == null) + { + tableStructure.setIsMain(true); + tableStructure.setIsMany(false); + tableStructure.setAssociationPath(null); + } + else + { + tableStructure.setIsMain(false); + + QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) + { + tableStructure.setIsMany(true); + } + + for(JoinOn joinOn : join.getJoinOns()) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // don't allow the user to map the "join field" from a child up to its parent // + // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // + //////////////////////////////////////////////////////////////////////////////////////////////// + if(join.getLeftTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); + } + else if(join.getRightTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getRightField()); + } + } + + if(!StringUtils.hasContent(parentAssociationPath)) + { + tableStructure.setAssociationPath(association.getName()); + } + else + { + tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); + } + } + + ArrayList fields = new ArrayList<>(); + tableStructure.setFields(fields); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) + { + fields.add(field); + } + } + + fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), ""))); + + for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // at this time, we are not prepared to handle 3-level deep associations, so, only process them from the top level... // + // main challenge being, wide-mode. so, maybe we should just only support 3-level+ associations for tall? // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(association == null) + { + String nextLevelPath = + (StringUtils.hasContent(parentAssociationPath) ? parentAssociationPath + "." : "") + + (association != null ? association.getName() : ""); + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath); + tableStructure.addAssociation(associatedStructure); + } + } + + return (tableStructure); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java new file mode 100644 index 00000000..a0f90110 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -0,0 +1,325 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.collections.CaseInsensitiveKeyMap; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadValueMapper +{ + private static final QLogger LOG = QLogger.getLogger(BulkLoadValueMapper.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException + { + valueMapping(records, mapping, table, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; + String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; + + Map> possibleValueToRecordMap = new HashMap<>(); + + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); + for(QRecord record : records) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // if we remove in the loop, we get ConcurrentModificationException, so track which ones to remove // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + Set toRemove = new HashSet<>(); + + for(Map.Entry valueEntry : record.getValues().entrySet()) + { + QFieldMetaData field = table.getField(valueEntry.getKey()); + Serializable value = valueEntry.getValue(); + + String fieldNamePlusWideIndex = field.getName(); + if(record.getBackendDetail("wideAssociationIndexes") != null) + { + ArrayList indexes = (ArrayList) record.getBackendDetail("wideAssociationIndexes"); + fieldNamePlusWideIndex += "," + StringUtils.join(",", indexes); + } + + /////////////////// + // value mappin' // + /////////////////// + if(mappingForTable.containsKey(fieldNamePlusWideIndex) && value != null) + { + Serializable mappedValue = mappingForTable.get(fieldNamePlusWideIndex).get(ValueUtils.getValueAsString(value)); + if(mappedValue != null) + { + value = mappedValue; + } + } + + ///////////////////// + // type convertin' // + ///////////////////// + if(value != null && !"".equals(value)) + { + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + ListingHash fieldPossibleValueToRecordMap = possibleValueToRecordMap.computeIfAbsent(field.getName(), k -> new ListingHash<>()); + fieldPossibleValueToRecordMap.add(ValueUtils.getValueAsString(value), record); + } + else + { + QFieldType type = field.getType(); + try + { + value = ValueUtils.getValueAsFieldType(type, value); + record.setValue(field.getName(), value); + } + catch(Exception e) + { + record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); + toRemove.add(field.getName()); + } + } + } + } + + ////////////////////////////////////////////////////////////////////////////////////////// + // remove any field values that had an error. // + // otherwise there can be downstream issues (e.g., if say some customizer code tries to // + // build an entity out of a record, and there's a temporal value that can't be parsed. // + ////////////////////////////////////////////////////////////////////////////////////////// + toRemove.forEach(record::removeValue); + + ////////////////////////////////////// + // recursively process associations // + ////////////////////////////////////// + for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) + { + String associationName = entry.getKey(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } + } + } + + ////////////////////////////////////////// + // look up and validate possible values // + ////////////////////////////////////////// + for(Map.Entry> entry : possibleValueToRecordMap.entrySet()) + { + String fieldName = entry.getKey(); + QFieldMetaData field = table.getField(fieldName); + ListingHash fieldPossibleValueToRecordMap = possibleValueToRecordMap.get(fieldName); + + handlePossibleValues(field, fieldPossibleValueToRecordMap, associationNamePrefixForFields, tableLabelPrefix); + } + } + + + + /*************************************************************************** + ** Given a listingHash of Strings from the bulk-load file, to QRecords, + ** we will either: + ** - make sure the value set for the field is valid in the PV type + ** - or put an error in the record (leaving the original value from the file in the field) + ** + ** We'll do potentially 2 possible-value searches - the first "by id" - + ** type-converting the input strings to the PV's id type. Then, if any + ** values weren't found by id, a second search by "labels"... which might + ** be a bit suspicious, e.g., if the PV has a multi-field label... + ***************************************************************************/ + private static void handlePossibleValues(QFieldMetaData field, ListingHash fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException + { + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); + + //////////////////////////////////////////////////////////// + // String from file -> List that have that value // + //////////////////////////////////////////////////////////// + Set values = fieldPossibleValueToRecordMap.keySet(); + + ///////////////////////////////////////////////////////////////////////////////// + // String from file -> Integer (for example) - that is the type-converted // + // version of the PVS's idType (but before any lookups were done with that id) // + // e.g., "42" -> 42 // + // e.g., "SOME_CONST" -> "SOME_CONST" (for PVS w/ string ids) // + ///////////////////////////////////////////////////////////////////////////////// + Map valuesToValueInPvsIdTypeMap = new HashMap<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // String versions of EITHER ids or values found in searchPossibleValueSource call (depending on what was searched by) // + // e.g., "42" -> QPossibleValue(42, "Forty-two") (when searched by id) // + // e.g., "Forty-Two" -> QPossibleValue(42, "Forty-two") (when searched by label) // + // e.g., "SOME_CONST" -> QPossibleValue("SOME_CONST", "Some Const") (when searched by id) // + // e.g., "Some Const" -> QPossibleValue("SOME_CONST", "Some Const") (when searched by label) // + // goal being - file could have "42" or "Forty-Two" (or "forty two") and those would all map to QPossibleValue(42, "Forty-two") // + // or - file could have "SOME_CONST" or "Some Const" (or "some const") and those would all map to QPossibleValue("SOME_CONST", "Some Const") // + // this is also why using CaseInsensitiveKeyMap! // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + CaseInsensitiveKeyMap> valuesFoundAsStrings = new CaseInsensitiveKeyMap<>(); + + ///////////////////////////////////////////////////////// + // String values (from file) that still need looked up // + ///////////////////////////////////////////////////////// + Set valuesNotFound = new HashSet<>(); + + //////////////////////////////////////////////////////// + // do a search, trying to use all given values as ids // + //////////////////////////////////////////////////////// + ArrayList idList = new ArrayList<>(); + SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); + searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); + + for(String value : values) + { + Serializable valueInPvsIdType = value; + + try + { + valueInPvsIdType = ValueUtils.getValueAsFieldType(possibleValueSource.getIdType(), value); + } + catch(Exception e) + { + //////////////////////////// + // leave as original type // + //////////////////////////// + } + + valuesToValueInPvsIdTypeMap.put(value, valueInPvsIdType); + idList.add(valueInPvsIdType); + valuesNotFound.add(value); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - we should probably be doing a lot of what QJavalinImplementation.finishPossibleValuesRequest does here // + // to apply possible-value filters. difficult to pass values in, but needed... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + searchPossibleValueSourceInput.setIdList(idList); + searchPossibleValueSourceInput.setLimit(values.size()); + LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0))); + SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = idList.isEmpty() ? new SearchPossibleValueSourceOutput() : new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // for each possible value found, store it as a hit, and remove it from the set of ones not-found // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) + { + String valueAsString = ValueUtils.getValueAsString(possibleValue.getId()); + valuesFoundAsStrings.put(valueAsString, possibleValue); + valuesNotFound.remove(valueAsString); + } + + /////////////////////////////////////////////////////////////////////////// + // if there are any that weren't found, try to look them up now by label // + /////////////////////////////////////////////////////////////////////////// + if(!valuesNotFound.isEmpty()) + { + searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); + searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); + searchPossibleValueSourceInput.setLabelList(new ArrayList<>(valuesNotFound)); + searchPossibleValueSourceInput.setLimit(valuesNotFound.size() * 10); // todo - a little sus... leaves some room for dupes, which, can they happen? + + LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", valuesNotFound.size()), logPair("firstLabel", () -> valuesNotFound.iterator().next())); + searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) + { + // todo - deal with multiple values found - and maybe... if some end up not-found, but some dupes happened, should we try another search, in case we hit the limit? + valuesFoundAsStrings.put(possibleValue.getLabel(), possibleValue); + valuesNotFound.remove(possibleValue.getLabel()); + } + } + + //////////////////////////////////////////////////////////////////////////////// + // for each record now, either set a usable value (e.g., a PV.id) or an error // + //////////////////////////////////////////////////////////////////////////////// + for(Map.Entry> entry : fieldPossibleValueToRecordMap.entrySet()) + { + String value = entry.getKey(); + Serializable valueInPvsIdType = valuesToValueInPvsIdTypeMap.get(entry.getKey()); + String pvsIdAsString = ValueUtils.getValueAsString(valueInPvsIdType); + + for(QRecord record : entry.getValue()) + { + if(valuesFoundAsStrings.containsKey(pvsIdAsString)) + { + record.setValue(field.getName(), valuesFoundAsStrings.get(pvsIdAsString).getId()); + } + else + { + record.addError(new BulkLoadPossibleValueError(associationNamePrefixForFields + field.getName(), value, tableLabelPrefix + field.getLabel())); + record.removeValue(field.getName()); + } + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java new file mode 100644 index 00000000..5add5f9b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; + + +/******************************************************************************* + ** Specialized error for records, for bulk-load use-cases, where we want to + ** report back info to the user about the field & value. + *******************************************************************************/ +public class BulkLoadValueTypeError extends AbstractBulkLoadRollableValueError +{ + private final String fieldLabel; + private final Serializable value; + private final QFieldType type; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel) + { + super("Cannot convert value [" + value + "] for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); + this.value = value; + this.type = type; + this.fieldLabel = fieldLabel; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getMessageToUseAsProcessSummaryRollupKey() + { + return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); + } + + + + /******************************************************************************* + ** Getter for value + ** + *******************************************************************************/ + @Override + public Serializable getValue() + { + return value; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java new file mode 100644 index 00000000..1da11588 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FlatRowsToRecord implements RowsToRecordInterface +{ + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + + boolean anyValuesFromFileUsed = false; + for(QFieldMetaData field : table.getFields().values()) + { + if(setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()))) + { + anyValuesFromFileUsed = true; + } + } + + ////////////////////////////////////////////////////////////////////////// + // avoid building empty records (e.g., "past the end" of an Excel file) // + ////////////////////////////////////////////////////////////////////////// + if(anyValuesFromFileUsed) + { + rs.add(record); + } + } + + BulkLoadValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java new file mode 100644 index 00000000..6d10b0bb --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -0,0 +1,103 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface RowsToRecordInterface +{ + + /*************************************************************************** + ** + ***************************************************************************/ + List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException; + + + /*************************************************************************** + ** returns true if value from row was used, else false. + ***************************************************************************/ + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex) + { + return setValueOrDefault(record, field, associationNameChain, mapping, row, columnIndex, null); + } + + /*************************************************************************** + ** returns true if value from row was used, else false. + ***************************************************************************/ + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex, List wideAssociationIndexes) + { + boolean valueFromRowWasUsed = false; + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // build full field-name -- possibly associations, then field name, then possibly index-suffix // + ///////////////////////////////////////////////////////////////////////////////////////////////// + String fieldName = field.getName(); + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + String fullFieldName = fieldNameWithAssociationPrefix + wideAssociationSuffix; + + ////////////////////////////////////////////// + // ok - look in the row - then the defaults // + ////////////////////////////////////////////// + Serializable value = null; + if(columnIndex != null && row != null) + { + value = row.getValueElseNull(columnIndex); + if(value != null && !"".equals(value)) + { + valueFromRowWasUsed = true; + } + } + else if(mapping.getFieldNameToDefaultValueMap().containsKey(fullFieldName)) + { + value = mapping.getFieldNameToDefaultValueMap().get(fullFieldName); + } + + if(value != null && !"".equals(value)) + { + record.setValue(fieldName, value); + } + + return (valueFromRowWasUsed); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java new file mode 100644 index 00000000..9aab3ab8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -0,0 +1,369 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import com.google.gson.reflect.TypeToken; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TallRowsToRecord implements RowsToRecordInterface +{ + private static final QLogger LOG = QLogger.getLogger(TallRowsToRecord.class); + + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + private Memoization> groupByAllIndexesFromTableMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + ArrayList rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; + + String associationNameChain = ""; + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null); + } + + //////////////////////// + // this is suspect... // + //////////////////////// + List rowGroupByValues = getGroupByValues(row, groupByIndexes); + if(rowGroupByValues == null) + { + continue; + } + + /////////////////////////////////////////////////////////////////////////////// + // maybe todo - some version of - only do this if there are mapped children? // + /////////////////////////////////////////////////////////////////////////////// + + if(rowsForCurrentRecord.isEmpty()) + { + /////////////////////////////////// + // this is first - so it's a yes // + /////////////////////////////////// + recordGroupByValues = rowGroupByValues; + rowsForCurrentRecord.add(row); + } + else if(Objects.equals(recordGroupByValues, rowGroupByValues)) + { + ///////////////////////////// + // a match - so keep going // + ///////////////////////////// + rowsForCurrentRecord.add(row); + } + else + { + ////////////////////////////////////////////////////////////// + // not first, and not a match, so we can finish this record // + ////////////////////////////////////////////////////////////// + QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord); + rs.add(record); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + fileToRowsInterface.unNext(); + + //////////////////////////////////////// + // reset these record-specific values // + //////////////////////////////////////// + rowsForCurrentRecord = new ArrayList<>(); + recordGroupByValues = null; + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // i wrote this condition in here: && rs.size() < limit // + // but IJ is saying it's always true... I can't quite see it, but, trusting static analysis... // + ///////////////////////////////////////////////////////////////////////////////////////////////// + if(!rowsForCurrentRecord.isEmpty()) + { + QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord); + rs.add(record); + } + + BulkLoadValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List groupByAllIndexesFromTable(BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow headerRow, String name) throws QException + { + return ((groupByAllIndexesFromTableMemoization.getResult(table.getName(), (n) -> + { + Map fieldIndexes = mapping.getFieldIndexes(table, name, headerRow); + return new ArrayList<>(fieldIndexes.values()); + })).orElse(null)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException + { + QRecord record = new QRecord(); + record.setTableName(table.getName()); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, CollectionUtils.useOrWrap(rows, new TypeToken>() {})); + + Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); + + //////////////////////////////////////////////////////// + // get all values for the main table from the 0th row // + //////////////////////////////////////////////////////// + BulkLoadFileRow row = rows.get(0); + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + } + + ///////////////////////////// + // associations (children) // + ///////////////////////////// + for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations())) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + List associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, headerRow, rows); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + + return record; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException + { + List rs = new ArrayList<>(); + + QTableMetaData table = QContext.getQInstance().getTable(associatedTable.getName()); + String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName; + + List rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; + for(BulkLoadFileRow row : rows) + { + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls); + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, associationNameChainForRecursiveCalls); + // throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); + } + + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // special case here - if there are no group-by-indexes for the row, it means there are no fields coming from columns in the file. // + // but, if any fields for this association have a default value - then - make a row using just default values. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.info("Handling case of an association with no fields from the file, but rather only defaults", logPair("associationName", associationName)); + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, List.of(row))); + break; + } + + /////////////////////////////////////////////////////////////////////////////// + // maybe todo - some version of - only do this if there are mapped children? // + /////////////////////////////////////////////////////////////////////////////// + + List rowGroupByValues = getGroupByValues(row, groupByIndexes); + if(rowGroupByValues == null) + { + continue; + } + + if(rowsForCurrentRecord.isEmpty()) + { + /////////////////////////////////// + // this is first - so it's a yes // + /////////////////////////////////// + recordGroupByValues = rowGroupByValues; + rowsForCurrentRecord.add(row); + } + else if(Objects.equals(recordGroupByValues, rowGroupByValues)) + { + ///////////////////////////// + // a match - so keep going // + ///////////////////////////// + rowsForCurrentRecord.add(row); + } + else + { + ////////////////////////////////////////////////////////////// + // not first, and not a match, so we can finish this record // + ////////////////////////////////////////////////////////////// + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord)); + + //////////////////////////////////////// + // reset these record-specific values // + //////////////////////////////////////// + rowsForCurrentRecord = new ArrayList<>(); + + ////////////////////////////////////////////////// + // use the current row to start the next record // + ////////////////////////////////////////////////// + rowsForCurrentRecord.add(row); + recordGroupByValues = rowGroupByValues; + } + } + + /////////// + // final // + /////////// + if(!rowsForCurrentRecord.isEmpty()) + { + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord)); + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getGroupByValues(BulkLoadFileRow row, List indexes) + { + List rowGroupByValues = new ArrayList<>(); + boolean haveAnyGroupByValues = false; + for(Integer index : indexes) + { + Serializable value = row.getValueElseNull(index); + rowGroupByValues.add(value); + + if(value != null && !"".equals(value)) + { + haveAnyGroupByValues = true; + } + } + + if(!haveAnyGroupByValues) + { + return (null); + } + + return (rowGroupByValues); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java new file mode 100644 index 00000000..2767c061 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -0,0 +1,274 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.google.gson.reflect.TypeToken; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** use a flatter mapping object, where field names look like: + ** associationChain.fieldName,index.subIndex + *******************************************************************************/ +public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>(), false); + rs.add(record); + } + + BulkLoadValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** may return null, if there were no values in the row for this (sub-wide) record. + ** more specifically: + ** + ** the param `rowOfOnlyDefaultValues` - should be false for the header table, + ** and true for an association iff all mapped fields are using 'default values' + ** (e.g., not values from the file). + ** + ** So this method will return null, indicating "no child row to build" if: + ** - when doing a rowOfOnlyDefaultValues - only if there actually weren't any + ** default values, which, probably never happens! + ** - else (doing a row with at least 1 value from the file) - then, null is + ** returned if there were NO values from the file. + ** + ** The goal here is to support these cases: + ** + ** Case A (a row of not only-default-values): + ** - lineItem.sku,0 = column: sku1 + ** - lineItem.qty,0 = column: qty1 + ** - lineItem.lineNo,0 = Default: 1 + ** - lineItem.sku,1 = column: sku2 + ** - lineItem.qty,1 = column: qty2 + ** - lineItem.lineNo,1 = Default: 2 + ** then a file row with no values for sku2 & qty2 - we don't want a row + ** in that case (which would only have the default value of lineNo=2) + ** + ** Case B (a row of only-default-values): + ** - lineItem.sku,0 = column: sku1 + ** - lineItem.qty,0 = column: qty1 + ** - lineItem.lineNo,0 = Default: 1 + ** - lineItem.sku,1 = Default: SUPPLEMENT + ** - lineItem.qty,1 = Default: 1 + ** - lineItem.lineNo,1 = Default: 2 + ** we want every parent (order) to include a 2nd line item - with 3 + ** default values (sku=SUPPLEMENT, qty=q, lineNo=2). + ** + ***************************************************************************/ + private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes, boolean rowOfOnlyDefaultValues) throws QException + { + ////////////////////////////////////////////////////// + // start by building the record with its own fields // + ////////////////////////////////////////////////////// + QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + + boolean hadAnyValuesInRowFromFile = false; + boolean hadAnyValuesInRow = false; + for(QFieldMetaData field : table.getFields().values()) + { + hadAnyValuesInRowFromFile = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRowFromFile; + + ///////////////////////////////////////////////////////////////////////////////////// + // for wide mode (different from tall) - allow a row that only has default values. // + // e.g., Line Item (2) might be a default to add to every order // + ///////////////////////////////////////////////////////////////////////////////////// + if(record.getValue(field.getName()) != null) + { + hadAnyValuesInRow = true; + } + } + + if(rowOfOnlyDefaultValues) + { + if(!hadAnyValuesInRow) + { + return (null); + } + } + else + { + if(!hadAnyValuesInRowFromFile) + { + return (null); + } + } + + ///////////////////////////// + // associations (children) // + ///////////////////////////// + for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations())) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + List associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, row, headerRow); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // stash the wide-association indexes in records, so that in the value mapper, we know if if this is, e.g., ,1, or ,2.3, for value-mapping // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + ArrayList indexesArrayList = CollectionUtils.useOrWrap(wideAssociationIndexes, new TypeToken<>() {}); + record.addBackendDetail("wideAssociationIndexes", indexesArrayList); + } + + return record; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException + { + List rs = new ArrayList<>(); + + String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName; + + for(int i = 0; true; i++) + { + // todo - doesn't support grand-children + List wideAssociationIndexes = List.of(i); + Map fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes); + + boolean rowOfOnlyDefaultValues = false; + if(fieldIndexes.isEmpty()) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there aren't any field-indexes for this (i) value (e.g., no columns mapped for Line Item: X (2)), we can still build a // + // child record here if there are any default values - so check for them - and only if they are empty, then break the loop. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Map fieldDefaultValues = mapping.getFieldDefaultValues(associatedTable, associationNameChainForRecursiveCalls, wideAssociationIndexes); + if(!CollectionUtils.nullSafeHasContents(fieldDefaultValues)) + { + break; + } + rowOfOnlyDefaultValues = true; + } + + QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes, rowOfOnlyDefaultValues); + if(record != null) + { + rs.add(record); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java new file mode 100644 index 00000000..e39e95c0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -0,0 +1,268 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); + } + + processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size()); + + rs.add(record); + } + + BulkLoadValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processAssociations(String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record, int startIndex, int endIndex) throws QException + { + for(String associationName : mapping.getMappedAssociations()) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + // List associatedRecords = processAssociation(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record); + List associatedRecords = processAssociationV2(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record, startIndex, endIndex); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException + { + List rs = new ArrayList<>(); + + Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + { + if(entry.getKey().startsWith(associationName + ".")) + { + String fieldName = entry.getKey().substring(associationName.length() + 1); + + ////////////////////////////////////////////////////////////////////////// + // make sure the name here is for this table - not a sub-table under it // + ////////////////////////////////////////////////////////////////////////// + if(!fieldName.contains(".")) + { + fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); + } + } + } + + ///////////////////////////////////////////////////////////////////// + // loop over the length of the record, building associated records // + ///////////////////////////////////////////////////////////////////// + QRecord associatedRecord = new QRecord(); + Set processedFieldNames = new HashSet<>(); + boolean gotAnyValues = false; + int subStartIndex = -1; + + for(int i = startIndex; i < endIndex; i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + + for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + { + if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + { + /////////////////////////////////////////////// + // ok - this is a value for this association // + /////////////////////////////////////////////// + if(subStartIndex == -1) + { + subStartIndex = i; + } + + String fieldName = entry.getKey(); + if(processedFieldNames.contains(fieldName)) + { + ///////////////////////////////////////////////// + // this means we're starting a new sub-record! // + ///////////////////////////////////////////////// + if(gotAnyValues) + { + addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); + rs.add(associatedRecord); + } + + associatedRecord = new QRecord(); + processedFieldNames = new HashSet<>(); + gotAnyValues = false; + subStartIndex = i + 1; + } + + processedFieldNames.add(fieldName); + + Serializable value = row.getValueElseNull(i); + if(value != null && !"".equals(value)) + { + gotAnyValues = true; + } + + setValueOrDefault(associatedRecord, table.getField(fieldName), associationName, mapping, row, i); + } + } + } + + //////////////////////// + // handle final value // + //////////////////////// + if(gotAnyValues) + { + addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); + rs.add(associatedRecord); + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) + { + for(QFieldMetaData field : table.getFields().values()) + { + if(!processedFieldNames.contains(field.getName())) + { + setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java new file mode 100644 index 00000000..e391b5f9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -0,0 +1,615 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.FlatRowsToRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.TallRowsToRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertMapping implements Serializable +{ + private String tableName; + private Boolean hasHeaderRow; + + private Layout layout; + + ///////////////////////////////////////////////////////////////////// + // keys in here are: // + // fieldName (for the main table) // + // association.fieldName (for an associated child table) // + // association.association.fieldName (for grandchild associations) // + ///////////////////////////////////////////////////////////////////// + private Map fieldNameToHeaderNameMap = new HashMap<>(); + private Map fieldNameToIndexMap = new HashMap<>(); + private Map fieldNameToDefaultValueMap = new HashMap<>(); + private Map> fieldNameToValueMapping = new HashMap<>(); + + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); + + private List mappedAssociations = new ArrayList<>(); + + private Memoization, Boolean> shouldProcessFieldForTable = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public enum Layout implements PossibleValueEnum + { + FLAT(FlatRowsToRecord::new), + TALL(TallRowsToRecord::new), + WIDE(WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping::new); + + + /*************************************************************************** + ** + ***************************************************************************/ + private final Supplier supplier; + + + + /*************************************************************************** + ** + ***************************************************************************/ + Layout(Supplier supplier) + { + this.supplier = supplier; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public RowsToRecordInterface newRowsToRecordInterface() + { + return (supplier.get()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueId() + { + return name(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return StringUtils.ucFirst(name().toLowerCase()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException + { + return getFieldIndexes(table, associationNameChain, headerRow, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException + { + if(hasHeaderRow && fieldNameToHeaderNameMap != null) + { + return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow, wideAssociationIndexes)); + } + else if(fieldNameToIndexMap != null) + { + return (getFieldIndexesForNoHeaderUseCase(table, associationNameChain, wideAssociationIndexes)); + } + + throw (new QException("Mapping was not properly configured.")); + } + + + + /*************************************************************************** + ** get a map of default-values for fields in a given table (at the specified + ** association chain and wide-indexes). Will only include fields using a + ** default value. + ***************************************************************************/ + @JsonIgnore + public Map getFieldDefaultValues(QTableMetaData table, String associationNameChain, List wideAssociationIndexes) throws QException + { + Map rs = new HashMap<>(); + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + /////////////////////////////////////////////////////////////////////////// + // loop over fields - adding them to the rs if they have a default value // + /////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + Serializable defaultValue = fieldNameToDefaultValueMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(defaultValue != null) + { + rs.put(field.getName(), defaultValue); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + private Map getFieldIndexesForNoHeaderUseCase(QTableMetaData table, String associationNameChain, List wideAssociationIndexes) + { + Map rs = new HashMap<>(); + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + Integer index = fieldNameToIndexMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(index != null) + { + rs.put(field.getName(), index); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map> getFieldNameToValueMappingForTable(String associatedTableName) + { + Map> rs = new HashMap<>(); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(fieldNameToValueMapping).entrySet()) + { + if(shouldProcessFieldForTable(entry.getKey(), associatedTableName)) + { + String key = StringUtils.hasContent(associatedTableName) ? entry.getKey().substring(associatedTableName.length() + 1) : entry.getKey(); + rs.put(key, entry.getValue()); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessFieldForTable(String fieldNameWithAssociationPrefix, String associationChain) + { + return shouldProcessFieldForTable.getResult(Pair.of(fieldNameWithAssociationPrefix, associationChain), p -> + { + List fieldNameParts = new ArrayList<>(); + List associationParts = new ArrayList<>(); + + if(StringUtils.hasContent(fieldNameWithAssociationPrefix)) + { + fieldNameParts.addAll(Arrays.asList(fieldNameWithAssociationPrefix.split("\\."))); + } + + if(StringUtils.hasContent(associationChain)) + { + associationParts.addAll(Arrays.asList(associationChain.split("\\."))); + } + + if(!fieldNameParts.isEmpty()) + { + fieldNameParts.remove(fieldNameParts.size() - 1); + } + + return (fieldNameParts.equals(associationParts)); + }).orElse(false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List wideAssociationIndexes) + { + Map rs = new HashMap<>(); + + //////////////////////////////////////////////////////// + // for the current file, map header values to indexes // + //////////////////////////////////////////////////////// + Map headerToIndexMap = new HashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + headerToIndexMap.put(headerValue, i); + } + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(headerName != null) + { + Integer headerIndex = headerToIndexMap.get(headerName); + if(headerIndex != null) + { + rs.put(field.getName(), headerIndex); + } + } + } + + return (rs); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public BulkInsertMapping withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for hasHeaderRow + *******************************************************************************/ + public Boolean getHasHeaderRow() + { + return (this.hasHeaderRow); + } + + + + /******************************************************************************* + ** Setter for hasHeaderRow + *******************************************************************************/ + public void setHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + } + + + + /******************************************************************************* + ** Fluent setter for hasHeaderRow + *******************************************************************************/ + public BulkInsertMapping withHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToHeaderNameMap + *******************************************************************************/ + public Map getFieldNameToHeaderNameMap() + { + return (this.fieldNameToHeaderNameMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToHeaderNameMap + *******************************************************************************/ + public void setFieldNameToHeaderNameMap(Map fieldNameToHeaderNameMap) + { + this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToHeaderNameMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToHeaderNameMap(Map fieldNameToHeaderNameMap) + { + this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToIndexMap + *******************************************************************************/ + public Map getFieldNameToIndexMap() + { + return (this.fieldNameToIndexMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToIndexMap + *******************************************************************************/ + public void setFieldNameToIndexMap(Map fieldNameToIndexMap) + { + this.fieldNameToIndexMap = fieldNameToIndexMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToIndexMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToIndexMap(Map fieldNameToIndexMap) + { + this.fieldNameToIndexMap = fieldNameToIndexMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for mappedAssociations + *******************************************************************************/ + public List getMappedAssociations() + { + return (this.mappedAssociations); + } + + + + /******************************************************************************* + ** Setter for mappedAssociations + *******************************************************************************/ + public void setMappedAssociations(List mappedAssociations) + { + this.mappedAssociations = mappedAssociations; + } + + + + /******************************************************************************* + ** Fluent setter for mappedAssociations + *******************************************************************************/ + public BulkInsertMapping withMappedAssociations(List mappedAssociations) + { + this.mappedAssociations = mappedAssociations; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToDefaultValueMap + *******************************************************************************/ + public Map getFieldNameToDefaultValueMap() + { + if(this.fieldNameToDefaultValueMap == null) + { + this.fieldNameToDefaultValueMap = new HashMap<>(); + } + + return (this.fieldNameToDefaultValueMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToDefaultValueMap + *******************************************************************************/ + public void setFieldNameToDefaultValueMap(Map fieldNameToDefaultValueMap) + { + this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToDefaultValueMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToDefaultValueMap(Map fieldNameToDefaultValueMap) + { + this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToValueMapping + *******************************************************************************/ + public Map> getFieldNameToValueMapping() + { + return (this.fieldNameToValueMapping); + } + + + + /******************************************************************************* + ** Setter for fieldNameToValueMapping + *******************************************************************************/ + public void setFieldNameToValueMapping(Map> fieldNameToValueMapping) + { + this.fieldNameToValueMapping = fieldNameToValueMapping; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToValueMapping + *******************************************************************************/ + public BulkInsertMapping withFieldNameToValueMapping(Map> fieldNameToValueMapping) + { + this.fieldNameToValueMapping = fieldNameToValueMapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for layout + *******************************************************************************/ + public Layout getLayout() + { + return (this.layout); + } + + + + /******************************************************************************* + ** Setter for layout + *******************************************************************************/ + public void setLayout(Layout layout) + { + this.layout = layout; + } + + + + /******************************************************************************* + ** Fluent setter for layout + *******************************************************************************/ + public BulkInsertMapping withLayout(Layout layout) + { + this.layout = layout; + return (this); + } + + + + /******************************************************************************* + ** Getter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public Map> getTallLayoutGroupByIndexMap() + { + return (this.tallLayoutGroupByIndexMap); + } + + + + /******************************************************************************* + ** Setter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public void setTallLayoutGroupByIndexMap(Map> tallLayoutGroupByIndexMap) + { + this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap; + } + + + + /******************************************************************************* + ** Fluent setter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public BulkInsertMapping withTallLayoutGroupByIndexMap(Map> tallLayoutGroupByIndexMap) + { + this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java new file mode 100644 index 00000000..6e383dd0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java @@ -0,0 +1,201 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + + +/******************************************************************************* + ** A row of values, e.g., from a file, for bulk-load + *******************************************************************************/ +public class BulkLoadFileRow implements Serializable +{ + private int rowNo; + private Serializable[] values; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadFileRow(Serializable[] values, int rowNo) + { + this.values = values; + this.rowNo = rowNo; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public int size() + { + if(values == null) + { + return (0); + } + + return (values.length); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public boolean hasIndex(int i) + { + if(values == null) + { + return (false); + } + + if(i >= values.length || i < 0) + { + return (false); + } + + return (true); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Serializable getValue(int i) + { + if(values == null) + { + throw new IllegalStateException("Row has no values"); + } + + if(i >= values.length || i < 0) + { + throw new IllegalArgumentException("Index out of bounds: Requested index " + i + "; values.length: " + values.length); + } + + return (values[i]); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Serializable getValueElseNull(int i) + { + if(!hasIndex(i)) + { + return (null); + } + + return (values[i]); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + if(values == null) + { + return ("null"); + } + + return Arrays.stream(values).map(String::valueOf).collect(Collectors.joining(",")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + + if(o == null || getClass() != o.getClass()) + { + return false; + } + + BulkLoadFileRow that = (BulkLoadFileRow) o; + return rowNo == that.rowNo && Objects.deepEquals(values, that.values); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public int hashCode() + { + return Objects.hash(rowNo, Arrays.hashCode(values)); + } + + + + /******************************************************************************* + ** Getter for rowNo + *******************************************************************************/ + public int getRowNo() + { + return (this.rowNo); + } + + + + /******************************************************************************* + ** Setter for rowNo + *******************************************************************************/ + public void setRowNo(int rowNo) + { + this.rowNo = rowNo; + } + + + + /******************************************************************************* + ** Fluent setter for rowNo + *******************************************************************************/ + public BulkLoadFileRow withRowNo(int rowNo) + { + this.rowNo = rowNo; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java new file mode 100644 index 00000000..2fe07b3c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java @@ -0,0 +1,165 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; + + +/*************************************************************************** + * this is the model of a saved bulk load profile - which is what passes back + * and forth with the frontend. + ****************************************************************************/ +public class BulkLoadProfile implements Serializable +{ + private ArrayList fieldList; + + private Boolean hasHeaderRow; + private String layout; + private String version; + + + + /******************************************************************************* + ** Getter for fieldList + *******************************************************************************/ + public ArrayList getFieldList() + { + return (this.fieldList); + } + + + + /******************************************************************************* + ** Getter for hasHeaderRow + *******************************************************************************/ + public Boolean getHasHeaderRow() + { + return (this.hasHeaderRow); + } + + + + /******************************************************************************* + ** Setter for hasHeaderRow + *******************************************************************************/ + public void setHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + } + + + + /******************************************************************************* + ** Fluent setter for hasHeaderRow + *******************************************************************************/ + public BulkLoadProfile withHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for layout + *******************************************************************************/ + public String getLayout() + { + return (this.layout); + } + + + + /******************************************************************************* + ** Setter for layout + *******************************************************************************/ + public void setLayout(String layout) + { + this.layout = layout; + } + + + + /******************************************************************************* + ** Fluent setter for layout + *******************************************************************************/ + public BulkLoadProfile withLayout(String layout) + { + this.layout = layout; + return (this); + } + + + + /******************************************************************************* + ** Setter for fieldList + *******************************************************************************/ + public void setFieldList(ArrayList fieldList) + { + this.fieldList = fieldList; + } + + + + /******************************************************************************* + ** Fluent setter for fieldList + *******************************************************************************/ + public BulkLoadProfile withFieldList(ArrayList fieldList) + { + this.fieldList = fieldList; + return (this); + } + + + /******************************************************************************* + ** Getter for version + *******************************************************************************/ + public String getVersion() + { + return (this.version); + } + + + + /******************************************************************************* + ** Setter for version + *******************************************************************************/ + public void setVersion(String version) + { + this.version = version; + } + + + + /******************************************************************************* + ** Fluent setter for version + *******************************************************************************/ + public BulkLoadProfile withVersion(String version) + { + this.version = version; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java new file mode 100644 index 00000000..f7ce0f6c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -0,0 +1,227 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.Map; + + +/*************************************************************************** + ** + ***************************************************************************/ +public class BulkLoadProfileField +{ + private String fieldName; + private Integer columnIndex; + private String headerName; + private Serializable defaultValue; + private Boolean doValueMapping; + private Map valueMappings; + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public BulkLoadProfileField withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnIndex + *******************************************************************************/ + public Integer getColumnIndex() + { + return (this.columnIndex); + } + + + + /******************************************************************************* + ** Setter for columnIndex + *******************************************************************************/ + public void setColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + } + + + + /******************************************************************************* + ** Fluent setter for columnIndex + *******************************************************************************/ + public BulkLoadProfileField withColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + return (this); + } + + + + /******************************************************************************* + ** Getter for defaultValue + *******************************************************************************/ + public Serializable getDefaultValue() + { + return (this.defaultValue); + } + + + + /******************************************************************************* + ** Setter for defaultValue + *******************************************************************************/ + public void setDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + } + + + + /******************************************************************************* + ** Fluent setter for defaultValue + *******************************************************************************/ + public BulkLoadProfileField withDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + return (this); + } + + + + /******************************************************************************* + ** Getter for doValueMapping + *******************************************************************************/ + public Boolean getDoValueMapping() + { + return (this.doValueMapping); + } + + + + /******************************************************************************* + ** Setter for doValueMapping + *******************************************************************************/ + public void setDoValueMapping(Boolean doValueMapping) + { + this.doValueMapping = doValueMapping; + } + + + + /******************************************************************************* + ** Fluent setter for doValueMapping + *******************************************************************************/ + public BulkLoadProfileField withDoValueMapping(Boolean doValueMapping) + { + this.doValueMapping = doValueMapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for valueMappings + *******************************************************************************/ + public Map getValueMappings() + { + return (this.valueMappings); + } + + + + /******************************************************************************* + ** Setter for valueMappings + *******************************************************************************/ + public void setValueMappings(Map valueMappings) + { + this.valueMappings = valueMappings; + } + + + + /******************************************************************************* + ** Fluent setter for valueMappings + *******************************************************************************/ + public BulkLoadProfileField withValueMappings(Map valueMappings) + { + this.valueMappings = valueMappings; + return (this); + } + + + /******************************************************************************* + ** Getter for headerName + *******************************************************************************/ + public String getHeaderName() + { + return (this.headerName); + } + + + + /******************************************************************************* + ** Setter for headerName + *******************************************************************************/ + public void setHeaderName(String headerName) + { + this.headerName = headerName; + } + + + + /******************************************************************************* + ** Fluent setter for headerName + *******************************************************************************/ + public BulkLoadProfileField withHeaderName(String headerName) + { + this.headerName = headerName; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java new file mode 100644 index 00000000..db55198f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java @@ -0,0 +1,275 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadTableStructure implements Serializable +{ + private boolean isMain; + private boolean isMany; + + private String tableName; + private String label; + private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild + + private ArrayList fields; // mmm, not marked as serializable (at this time) - is okay? + private ArrayList associations; + + + + /******************************************************************************* + ** Getter for isMain + *******************************************************************************/ + public boolean getIsMain() + { + return (this.isMain); + } + + + + /******************************************************************************* + ** Setter for isMain + *******************************************************************************/ + public void setIsMain(boolean isMain) + { + this.isMain = isMain; + } + + + + /******************************************************************************* + ** Fluent setter for isMain + *******************************************************************************/ + public BulkLoadTableStructure withIsMain(boolean isMain) + { + this.isMain = isMain; + return (this); + } + + + + /******************************************************************************* + ** Getter for isMany + *******************************************************************************/ + public boolean getIsMany() + { + return (this.isMany); + } + + + + /******************************************************************************* + ** Setter for isMany + *******************************************************************************/ + public void setIsMany(boolean isMany) + { + this.isMany = isMany; + } + + + + /******************************************************************************* + ** Fluent setter for isMany + *******************************************************************************/ + public BulkLoadTableStructure withIsMany(boolean isMany) + { + this.isMany = isMany; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public BulkLoadTableStructure withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public BulkLoadTableStructure withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields + *******************************************************************************/ + public ArrayList getFields() + { + return (this.fields); + } + + + + /******************************************************************************* + ** Setter for fields + *******************************************************************************/ + public void setFields(ArrayList fields) + { + this.fields = fields; + } + + + + /******************************************************************************* + ** Fluent setter for fields + *******************************************************************************/ + public BulkLoadTableStructure withFields(ArrayList fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for associationPath + *******************************************************************************/ + public String getAssociationPath() + { + return (this.associationPath); + } + + + + /******************************************************************************* + ** Setter for associationPath + *******************************************************************************/ + public void setAssociationPath(String associationPath) + { + this.associationPath = associationPath; + } + + + + /******************************************************************************* + ** Fluent setter for associationPath + *******************************************************************************/ + public BulkLoadTableStructure withAssociationPath(String associationPath) + { + this.associationPath = associationPath; + return (this); + } + + + + /******************************************************************************* + ** Getter for associations + *******************************************************************************/ + public ArrayList getAssociations() + { + return (this.associations); + } + + + + /******************************************************************************* + ** Setter for associations + *******************************************************************************/ + public void setAssociations(ArrayList associations) + { + this.associations = associations; + } + + + + /******************************************************************************* + ** Fluent setter for associations + *******************************************************************************/ + public BulkLoadTableStructure withAssociations(ArrayList associations) + { + this.associations = associations; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void addAssociation(BulkLoadTableStructure association) + { + if(this.associations == null) + { + this.associations = new ArrayList<>(); + } + this.associations.add(association); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index cbe1e1a9..47c8c036 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -239,6 +239,10 @@ public class ColumnStatsStep implements BackendStep QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(); qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // todo - be aware of possible name collisions here!! (e.g., a table w/ a field named `count`) // + ///////////////////////////////////////////////////////////////////////////////////////////////// QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts); runBackendStepOutput.addValue("valueCounts", valueCounts); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java index 0fcd5abf..f675f128 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; @@ -33,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp *******************************************************************************/ public class NoopLoadStep extends AbstractLoadStep { + private static final QLogger LOG = QLogger.getLogger(NoopLoadStep.class); /******************************************************************************* @@ -45,6 +47,7 @@ public class NoopLoadStep extends AbstractLoadStep /////////// // noop. // /////////// + LOG.trace("noop"); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 8d3e8287..09e621ce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -23,10 +23,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.Serializable; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -69,6 +73,8 @@ import com.kingsrook.qqq.backend.core.processes.implementations.basepull.Basepul *******************************************************************************/ public class StreamedETLWithFrontendProcess { + private static final QLogger LOG = QLogger.getLogger(StreamedETLWithFrontendProcess.class); + public static final String STEP_NAME_PREVIEW = "preview"; public static final String STEP_NAME_REVIEW = "review"; public static final String STEP_NAME_VALIDATE = "validate"; @@ -193,6 +199,28 @@ public class StreamedETLWithFrontendProcess } + /*************************************************************************** + ** useful for a process step to call upon 'back' + ***************************************************************************/ + public static void resetValidationFields(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + runBackendStepInput.addValue(FIELD_DO_FULL_VALIDATION, null); + runBackendStepInput.addValue(FIELD_VALIDATION_SUMMARY, null); + runBackendStepInput.addValue(FIELD_PROCESS_SUMMARY, null); + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // in case, on the first time forward, the review step got moved after the validation step // + // (see BaseStreamedETLStep.moveReviewStepAfterValidateStep) - then un-do that upon going back. // + ////////////////////////////////////////////////////////////////////////////////////////////////// + ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + LOG.debug("Resetting step list. It was:" + stepList); + stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); + stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_PREVIEW) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + runBackendStepOutput.getProcessState().setStepList(stepList); + LOG.debug("... and now step list is: " + stepList); + } + + /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index 606a026b..6296f31c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -477,4 +477,13 @@ public class ProcessSummaryWarningsAndErrorsRollup } + + /******************************************************************************* + ** Getter for errorSummaries + ** + *******************************************************************************/ + public Map getErrorSummaries() + { + return errorSummaries; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..e909751e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; + + +/******************************************************************************* + ** Process used by the delete bulkLoadProfile dialog + *******************************************************************************/ +public class DeleteSavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(DeleteSavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("deleteSavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(DeleteSavedBulkLoadProfileProcess.class)) + .withName("delete") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + + try + { + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("id"); + + DeleteInput input = new DeleteInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setPrimaryKeys(List.of(savedBulkLoadProfileId)); + new DeleteAction().execute(input); + } + catch(Exception e) + { + LOG.warn("Error deleting saved bulkLoadProfile", e); + throw (e); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..c1bdaa41 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java @@ -0,0 +1,129 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Process used by the saved bulkLoadProfile dialogs + *******************************************************************************/ +public class QuerySavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(QuerySavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("querySavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(QuerySavedBulkLoadProfileProcess.class)) + .withName("query") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("id"); + + try + { + if(savedBulkLoadProfileId != null) + { + GetInput input = new GetInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setPrimaryKey(savedBulkLoadProfileId); + + GetOutput output = new GetAction().execute(input); + if(output.getRecord() == null) + { + throw (new QNotFoundException("The requested bulkLoadProfile was not found.")); + } + + runBackendStepOutput.addRecord(output.getRecord()); + runBackendStepOutput.addValue("savedBulkLoadProfile", output.getRecord()); + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) List.of(output.getRecord())); + } + else + { + String tableName = runBackendStepInput.getValueString("tableName"); + + QueryInput input = new QueryInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)) + .withOrderBy(new QFilterOrderBy("label"))); + + QueryOutput output = new QueryAction().execute(input); + runBackendStepOutput.setRecords(output.getRecords()); + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) output.getRecords()); + } + } + catch(QNotFoundException qnfe) + { + LOG.info("BulkLoadProfile not found", logPair("savedBulkLoadProfileId", savedBulkLoadProfileId)); + throw (qnfe); + } + catch(Exception e) + { + LOG.warn("Error querying for saved bulkLoadProfiles", e); + throw (e); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..0d8f33d4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java @@ -0,0 +1,171 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Process used by the saved bulkLoadProfile dialog + *******************************************************************************/ +public class StoreSavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(StoreSavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("storeSavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(StoreSavedBulkLoadProfileProcess.class)) + .withName("store") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + + try + { + String userId = QContext.getQSession().getUser().getIdReference(); + String tableName = runBackendStepInput.getValueString("tableName"); + String label = runBackendStepInput.getValueString("label"); + + String mappingJson = processMappingJson(runBackendStepInput.getValueString("mappingJson")); + + QRecord qRecord = new QRecord() + .withValue("id", runBackendStepInput.getValueInteger("id")) + .withValue("mappingJson", mappingJson) + .withValue("label", label) + .withValue("tableName", tableName) + .withValue("userId", userId); + + List savedBulkLoadProfileList; + if(qRecord.getValueInteger("id") == null) + { + checkForDuplicates(userId, tableName, label, null); + + InsertInput input = new InsertInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setRecords(List.of(qRecord)); + + InsertOutput output = new InsertAction().execute(input); + savedBulkLoadProfileList = output.getRecords(); + } + else + { + checkForDuplicates(userId, tableName, label, qRecord.getValueInteger("id")); + + UpdateInput input = new UpdateInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setRecords(List.of(qRecord)); + + UpdateOutput output = new UpdateAction().execute(input); + savedBulkLoadProfileList = output.getRecords(); + } + + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) savedBulkLoadProfileList); + } + catch(Exception e) + { + LOG.warn("Error storing saved bulkLoadProfile", e); + throw (e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String processMappingJson(String mappingJson) + { + return mappingJson; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void checkForDuplicates(String userId, String tableName, String label, Integer id) throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(SavedBulkLoadProfile.TABLE_NAME); + queryInput.setFilter(new QQueryFilter( + new QFilterCriteria("userId", QCriteriaOperator.EQUALS, userId), + new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName), + new QFilterCriteria("label", QCriteriaOperator.EQUALS, label))); + + if(id != null) + { + queryInput.getFilter().addCriteria(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, id)); + } + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) + { + throw (new QUserFacingException("You already have a saved Bulk Load Profile on this table with this name.")); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java index f91a9000..900dc4da 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java @@ -92,16 +92,21 @@ public class RenderSavedReportExecuteStep implements BackendStep //////////////////////////////// // read inputs, set up params // //////////////////////////////// - String sesProviderName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.SES_PROVIDER_NAME); - String fromEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FROM_EMAIL_ADDRESS); - String replyToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.REPLY_TO_EMAIL_ADDRESS); - String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); - ReportFormat reportFormat = ReportFormat.fromString(runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT)); - String sendToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS); - String emailSubject = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT); - SavedReport savedReport = new SavedReport(runBackendStepInput.getRecords().get(0)); - String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport); - String storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension(); + String sesProviderName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.SES_PROVIDER_NAME); + String fromEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FROM_EMAIL_ADDRESS); + String replyToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.REPLY_TO_EMAIL_ADDRESS); + String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); + ReportFormat reportFormat = ReportFormat.fromString(runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT)); + String sendToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS); + String emailSubject = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT); + SavedReport savedReport = new SavedReport(runBackendStepInput.getRecords().get(0)); + + String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport); + String storageReference = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_REFERENCE); + if(!StringUtils.hasContent(storageReference)) + { + storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension(); + } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if sending an email (or emails), validate the addresses before doing anything so user gets error and can fix // @@ -241,7 +246,7 @@ public class RenderSavedReportExecuteStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - private String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report) + public static String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmm").withZone(ZoneId.systemDefault()); String datePart = formatter.format(Instant.now()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java index c8693195..4b25c61c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java @@ -56,6 +56,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf public static final String FROM_EMAIL_ADDRESS = "fromEmailAddress"; public static final String REPLY_TO_EMAIL_ADDRESS = "replyToEmailAddress"; public static final String FIELD_NAME_STORAGE_TABLE_NAME = "storageTableName"; + public static final String FIELD_NAME_STORAGE_REFERENCE = "storageReference"; public static final String FIELD_NAME_REPORT_FORMAT = "reportFormat"; public static final String FIELD_NAME_EMAIL_ADDRESS = "reportDestinationEmailAddress"; public static final String FIELD_NAME_EMAIL_SUBJECT = "emailSubject"; @@ -81,6 +82,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf .withField(new QFieldMetaData(FROM_EMAIL_ADDRESS, QFieldType.STRING)) .withField(new QFieldMetaData(REPLY_TO_EMAIL_ADDRESS, QFieldType.STRING)) .withField(new QFieldMetaData(FIELD_NAME_STORAGE_TABLE_NAME, QFieldType.STRING)) + .withField(new QFieldMetaData(FIELD_NAME_STORAGE_REFERENCE, QFieldType.STRING)) .withRecordListMetaData(new QRecordListMetaData().withTableName(SavedReport.TABLE_NAME))) .withCode(new QCodeReference(RenderSavedReportPreStep.class))) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java index 96a2e43f..b042360c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java @@ -101,7 +101,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // in case the app added a security field to the scripts table, make sure the user is allowed to edit the script // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ValidateRecordSecurityLockHelper.validateSecurityFields(QContext.getQInstance().getTable(Script.TABLE_NAME), List.of(script), ValidateRecordSecurityLockHelper.Action.UPDATE); + ValidateRecordSecurityLockHelper.validateSecurityFields(QContext.getQInstance().getTable(Script.TABLE_NAME), List.of(script), ValidateRecordSecurityLockHelper.Action.UPDATE, transaction); if(CollectionUtils.nullSafeHasContents(script.getErrors())) { throw (new QPermissionDeniedException(script.getErrors().get(0).getMessage())); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index b14847f7..c8f6b82e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -173,8 +173,21 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt *******************************************************************************/ protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List sourceKeyList) { - String destinationTableForeignKeyField = getSyncProcessConfig().destinationTableForeignKey; - return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList)); + String destinationTableForeignKeyFieldName = getSyncProcessConfig().destinationTableForeignKey; + String destinationTableName = getSyncProcessConfig().destinationTable; + QFieldMetaData destinationForeignKeyField = QContext.getQInstance().getTable(destinationTableName).getField(destinationTableForeignKeyFieldName); + + List sourceKeysInDestinationKeyTypeList = null; + if(sourceKeyList != null) + { + sourceKeysInDestinationKeyTypeList = new ArrayList<>(); + for(Serializable sourceKey : sourceKeyList) + { + sourceKeysInDestinationKeyTypeList.add(ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKey)); + } + } + + return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyFieldName, QCriteriaOperator.IN, sourceKeysInDestinationKeyTypeList)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java index 4e9b214a..c338fa97 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java @@ -95,7 +95,7 @@ public class ProcessLockMetaDataProducer implements MetaDataProducerInterface. + */ + +package com.kingsrook.qqq.backend.core.processes.locks; + + +/*************************************************************************** + ** Record to hold either a processLock, or an unableToObtainProcessLockException. + ** Used as return value from bulk-methods in ProcessLockUtils (where some + ** requested keys may succeed and return a lock, and others may fail + ** and return the exception). + ***************************************************************************/ +public record ProcessLockOrException(ProcessLock processLock, UnableToObtainProcessLockException unableToObtainProcessLockException) +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ProcessLockOrException(ProcessLock processLock) + { + this(processLock, null); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ProcessLockOrException(UnableToObtainProcessLockException unableToObtainProcessLockException) + { + this(null, unableToObtainProcessLockException); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java index ac3a05e8..0d062ce1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java @@ -22,14 +22,24 @@ package com.kingsrook.qqq.backend.core.processes.locks; +import java.io.Serializable; import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; @@ -39,6 +49,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -70,10 +84,79 @@ public class ProcessLockUtils /******************************************************************************* + ** try to create a process lock, of a given key & type - but immediately fail + ** if the lock already exists. ** + ** @param key along with typeName, part of Unique Key for the lock. + ** @param typeName along with key, part of Unique Key for the lock. Must be a + * defined lock type, from which we derive defaultExpirationSeconds. + ** @param details advice to show users re: who/what created the lock. *******************************************************************************/ public static ProcessLock create(String key, String typeName, String details) throws UnableToObtainProcessLockException, QException { + Map locks = createMany(List.of(key), typeName, details); + return getProcessLockOrThrow(key, locks); + } + + + + /******************************************************************************* + ** try to create a process lock, of a given key & type - and re-try if it failed. + ** (e.g., wait until existing lock holder releases the lock). + ** + ** @param key along with typeName, part of Unique Key for the lock. + ** @param typeName along with key, part of Unique Key for the lock. Must be a + * defined lock type, from which we derive defaultExpirationSeconds. + ** @param details advice to show users re: who/what created the lock. + ** @param sleepBetweenTries how long to sleep between retries. + ** @param maxWait max amount of that will be waited between call to this method + * and an eventual UnableToObtainProcessLockException (plus or minus + * one sleepBetweenTries (actually probably just plus that). + ** + *******************************************************************************/ + public static ProcessLock create(String key, String typeName, String details, Duration sleepBetweenTries, Duration maxWait) throws UnableToObtainProcessLockException, QException + { + Map locks = createMany(List.of(key), typeName, details, sleepBetweenTries, maxWait); + return getProcessLockOrThrow(key, locks); + } + + + + /*************************************************************************** + ** For the single-lock versions of create, either return the lock identified by + ** key, or throw. + ***************************************************************************/ + private static ProcessLock getProcessLockOrThrow(String key, Map locks) throws UnableToObtainProcessLockException + { + if(locks.get(key) != null && locks.get(key).processLock() != null) + { + return (locks.get(key).processLock()); + } + else if(locks.get(key) != null && locks.get(key).unableToObtainProcessLockException() != null) + { + throw (locks.get(key).unableToObtainProcessLockException()); + } + else + { + throw (new UnableToObtainProcessLockException("Missing key [" + key + "] in response from request to create lock. Lock not created.")); + } + } + + + + /******************************************************************************* + ** try to create many process locks, of list of keys & a type - but immediately + ** fail (on a one-by-one basis) if the lock already exists. + ** + ** @param keys along with typeName, part of Unique Key for the lock. + ** @param typeName along with key, part of Unique Key for the lock. Must be a + * defined lock type, from which we derive defaultExpirationSeconds. + ** @param details advice to show users re: who/what created the lock. + *******************************************************************************/ + public static Map createMany(List keys, String typeName, String details) throws QException + { + Map rs = new HashMap<>(); + ProcessLockType lockType = getProcessLockTypeByName(typeName); if(lockType == null) { @@ -82,137 +165,305 @@ public class ProcessLockUtils QSession qSession = QContext.getQSession(); - Instant now = Instant.now(); - ProcessLock processLock = new ProcessLock() - .withKey(key) - .withProcessLockTypeId(lockType.getId()) - .withSessionUUID(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUuid(), null)) - .withUserId(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUser().getIdReference(), null)) - .withDetails(details) - .withCheckInTimestamp(now); + Instant now = Instant.now(); + Integer defaultExpirationSeconds = lockType.getDefaultExpirationSeconds(); + List processLocksToInsert = new ArrayList<>(); - Integer defaultExpirationSeconds = lockType.getDefaultExpirationSeconds(); - if(defaultExpirationSeconds != null) + Function constructProcessLockFromKey = (key) -> { - processLock.setExpiresAtTimestamp(now.plusSeconds(defaultExpirationSeconds)); + ProcessLock processLock = new ProcessLock() + .withKey(key) + .withProcessLockTypeId(lockType.getId()) + .withSessionUUID(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUuid(), null)) + .withUserId(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUser().getIdReference(), null)) + .withDetails(details) + .withCheckInTimestamp(now); + + if(defaultExpirationSeconds != null) + { + processLock.setExpiresAtTimestamp(now.plusSeconds(defaultExpirationSeconds)); + } + + return (processLock); + }; + + for(String key : keys) + { + processLocksToInsert.add(constructProcessLockFromKey.apply(key)); } - QRecord insertOutputRecord = tryToInsert(processLock); + Map insertResultMap = tryToInsertMany(processLocksToInsert); - //////////////////////////////////////////////////////////// - // if inserting failed... see if we can get existing lock // - //////////////////////////////////////////////////////////// - StringBuilder existingLockDetails = new StringBuilder(); - ProcessLock existingLock = null; - if(CollectionUtils.nullSafeHasContents(insertOutputRecord.getErrors())) + //////////////////////////////////////// + // look at which (if any) keys failed // + //////////////////////////////////////// + Set failedKeys = new HashSet<>(); + for(Map.Entry entry : insertResultMap.entrySet()) { - QRecord existingLockRecord = new GetAction().executeForRecord(new GetInput(ProcessLock.TABLE_NAME).withUniqueKey(Map.of("key", key, "processLockTypeId", lockType.getId()))); - if(existingLockRecord != null) + if(entry.getValue().unableToObtainProcessLockException() != null) { - existingLock = new ProcessLock(existingLockRecord); - if(StringUtils.hasContent(existingLock.getUserId())) - { - existingLockDetails.append("Held by: ").append(existingLock.getUserId()); - } + failedKeys.add(entry.getKey()); + } + } - if(StringUtils.hasContent(existingLock.getDetails())) - { - existingLockDetails.append("; with details: ").append(existingLock.getDetails()); - } + ////////////////////////////////////////////////////////////////////// + // if any keys failed, try to get the existing locks for those keys // + ////////////////////////////////////////////////////////////////////// + Map existingLockRecords = new HashMap<>(); + if(CollectionUtils.nullSafeHasContents(failedKeys)) + { + QueryOutput queryOutput = new QueryAction().execute(new QueryInput(ProcessLock.TABLE_NAME).withFilter(new QQueryFilter() + .withCriteria("processLockTypeId", QCriteriaOperator.EQUALS, lockType.getId()) + .withCriteria("key", QCriteriaOperator.IN, failedKeys))); + for(QRecord record : queryOutput.getRecords()) + { + existingLockRecords.put(record.getValueString("key"), record); + } + } - Instant expiresAtTimestamp = existingLock.getExpiresAtTimestamp(); - if(expiresAtTimestamp != null) - { - ZonedDateTime zonedExpiresAt = expiresAtTimestamp.atZone(ValueUtils.getSessionOrInstanceZoneId()); - existingLockDetails.append("; expiring at: ").append(QValueFormatter.formatDateTimeWithZone(zonedExpiresAt)); - } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over results from insert call - either adding successes to the output structure, or adding details about failures, // + // OR - deleting expired locks and trying a second insert! // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List deleteIdList = new ArrayList<>(); + List tryAgainList = new ArrayList<>(); + Map existingLockDetailsMap = new HashMap<>(); + Map existingLockMap = new HashMap<>(); + for(Map.Entry entry : insertResultMap.entrySet()) + { + String key = entry.getKey(); + ProcessLock processLock = entry.getValue().processLock(); - if(expiresAtTimestamp != null && expiresAtTimestamp.isBefore(now)) - { - ///////////////////////////////////////////////////////////////////////////////// - // if existing lock has expired, then we can delete it and try to insert again // - ///////////////////////////////////////////////////////////////////////////////// - LOG.info("Existing lock has expired - deleting it and trying again.", logPair("id", existingLock.getId()), - logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", expiresAtTimestamp)); - new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKey(existingLock.getId())); - insertOutputRecord = tryToInsert(processLock); - } + ////////////////////////////////////////////////////////////////////////// + // if inserting failed... see if we found an existing lock for this key // + ////////////////////////////////////////////////////////////////////////// + StringBuilder existingLockDetails = new StringBuilder(); + ProcessLock existingLock = null; + + if(processLock != null) + { + rs.put(key, new ProcessLockOrException(processLock)); } else { - ///////////////////////////////////////////////////////// - // if existing lock doesn't exist, try to insert again // - ///////////////////////////////////////////////////////// - insertOutputRecord = tryToInsert(processLock); + QRecord existingLockRecord = existingLockRecords.get(key); + if(existingLockRecord != null) + { + existingLock = new ProcessLock(existingLockRecord); + if(StringUtils.hasContent(existingLock.getUserId())) + { + existingLockDetails.append("Held by: ").append(existingLock.getUserId()); + } + + if(StringUtils.hasContent(existingLock.getDetails())) + { + existingLockDetails.append("; with details: ").append(existingLock.getDetails()); + } + + Instant expiresAtTimestamp = existingLock.getExpiresAtTimestamp(); + if(expiresAtTimestamp != null) + { + ZonedDateTime zonedExpiresAt = expiresAtTimestamp.atZone(ValueUtils.getSessionOrInstanceZoneId()); + existingLockDetails.append("; expiring at: ").append(QValueFormatter.formatDateTimeWithZone(zonedExpiresAt)); + } + + existingLockDetailsMap.put(key, existingLockDetails.toString()); + existingLockMap.put(key, existingLock); + + if(expiresAtTimestamp != null && expiresAtTimestamp.isBefore(now)) + { + ///////////////////////////////////////////////////////////////////////////////// + // if existing lock has expired, then we can delete it and try to insert again // + ///////////////////////////////////////////////////////////////////////////////// + LOG.info("Existing lock has expired - deleting it and trying again.", logPair("id", existingLock.getId()), + logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", expiresAtTimestamp)); + deleteIdList.add(existingLock.getId()); + tryAgainList.add(constructProcessLockFromKey.apply(key)); + } + } + else + { + /////////////////////////////////////////////////////////////////////////////// + // if existing lock doesn't exist now (e.g., it was deleted before the UC // + // check failed and when we looked for it), then just try to insert it again // + /////////////////////////////////////////////////////////////////////////////// + tryAgainList.add(constructProcessLockFromKey.apply(key)); + } } } - if(CollectionUtils.nullSafeHasContents(insertOutputRecord.getErrors())) + ///////////////////////////////////////////////////// + // if there are expired locks to delete, do so now // + ///////////////////////////////////////////////////// + if(!deleteIdList.isEmpty()) { - ///////////////////////////////////////////////////////////////////////////////// - // if at this point, we have errors on the last attempted insert, then give up // - ///////////////////////////////////////////////////////////////////////////////// - LOG.info("Errors in process lock record after attempted insert", logPair("errors", insertOutputRecord.getErrors()), - logPair("key", key), logPair("type", typeName), logPair("details", details)); - throw (new UnableToObtainProcessLockException("A Process Lock already exists for key [" + key + "] of type [" + typeName + "], " + existingLockDetails) - .withExistingLock(existingLock)); + new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKeys(deleteIdList)); } - LOG.info("Created process lock", logPair("id", processLock.getId()), - logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", processLock.getExpiresAtTimestamp())); - return new ProcessLock(insertOutputRecord); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are any to try again (either because we just deleted their now-expired locks, or because we otherwise couldn't find their locks, do so now // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!tryAgainList.isEmpty()) + { + Map tryAgainResult = tryToInsertMany(tryAgainList); + for(Map.Entry entry : tryAgainResult.entrySet()) + { + String key = entry.getKey(); + ProcessLock processLock = entry.getValue().processLock(); + UnableToObtainProcessLockException unableToObtainProcessLockException = entry.getValue().unableToObtainProcessLockException(); + + if(processLock != null) + { + rs.put(key, new ProcessLockOrException(processLock)); + } + else + { + rs.put(key, new ProcessLockOrException(Objects.requireNonNullElseGet(unableToObtainProcessLockException, () -> new UnableToObtainProcessLockException("Process lock not created, but no details available.")))); + } + } + } + + //////////////////////////////////////////////////////////////////// + // put anything not successfully created into result map as error // + //////////////////////////////////////////////////////////////////// + for(ProcessLock processLock : processLocksToInsert) + { + String key = processLock.getKey(); + if(rs.containsKey(key)) + { + LOG.info("Created process lock", logPair("id", processLock.getId()), + logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", processLock.getExpiresAtTimestamp())); + } + else + { + if(existingLockDetailsMap.containsKey(key)) + { + rs.put(key, new ProcessLockOrException(new UnableToObtainProcessLockException("A Process Lock already exists for key [" + key + "] of type [" + typeName + "], " + existingLockDetailsMap.get(key)) + .withExistingLock(existingLockMap.get(key)))); + } + else + { + rs.put(key, new ProcessLockOrException(new UnableToObtainProcessLockException("Process lock for key [" + key + "] of type [" + typeName + "] was not created..."))); + } + } + } + + return (rs); } /******************************************************************************* - ** + ** Try to do an insert - noting that an exception from the InsertAction will be + ** caught in here, and placed in the records as an Error! *******************************************************************************/ - private static QRecord tryToInsert(ProcessLock processLock) throws QException + private static Map tryToInsertMany(List processLocks) { - return new InsertAction().execute(new InsertInput(ProcessLock.TABLE_NAME).withRecordEntity(processLock)).getRecords().get(0); + Map rs = new HashMap<>(); + + try + { + List insertedRecords = new InsertAction().execute(new InsertInput(ProcessLock.TABLE_NAME).withRecordEntities(processLocks)).getRecords(); + for(QRecord insertedRecord : insertedRecords) + { + String key = insertedRecord.getValueString("key"); + if(CollectionUtils.nullSafeHasContents(insertedRecord.getErrors())) + { + rs.put(key, new ProcessLockOrException(new UnableToObtainProcessLockException(insertedRecord.getErrors().get(0).getMessage()))); + } + else + { + rs.put(key, new ProcessLockOrException(new ProcessLock(insertedRecord))); + } + } + } + catch(Exception e) + { + for(ProcessLock processLock : processLocks) + { + rs.put(processLock.getKey(), new ProcessLockOrException(new UnableToObtainProcessLockException("Error attempting to insert process lock: " + e.getMessage()))); + } + } + + return (rs); } /******************************************************************************* + ** try to create many process locks, of a given list of key & a type - and re-try + ** upon failures (e.g., wait until existing lock holder releases the lock). + ** + ** @param keys along with typeName, part of Unique Key for the lock. + ** @param typeName along with key, part of Unique Key for the lock. Must be a + * defined lock type, from which we derive defaultExpirationSeconds. + ** @param details advice to show users re: who/what created the lock. + ** @param sleepBetweenTries how long to sleep between retries. + ** @param maxWait max amount of that will be waited between call to this method + * and an eventual UnableToObtainProcessLockException (plus or minus + * one sleepBetweenTries (actually probably just plus that). ** *******************************************************************************/ - public static ProcessLock create(String key, String type, String holderId, Duration sleepBetweenTries, Duration maxWait) throws UnableToObtainProcessLockException, QException + public static Map createMany(List keys, String typeName, String details, Duration sleepBetweenTries, Duration maxWait) throws QException { + Map rs = new HashMap<>(); + Map lastExceptionsPerKey = new HashMap<>(); + Set stillNeedCreated = new HashSet<>(keys); + Instant giveUpTime = Instant.now().plus(maxWait); UnableToObtainProcessLockException lastCaughtUnableToObtainProcessLockException = null; while(true) { - try + Map createManyResult = createMany(stillNeedCreated.size() == keys.size() ? keys : new ArrayList<>(stillNeedCreated), typeName, details); + for(Map.Entry entry : createManyResult.entrySet()) { - ProcessLock processLock = create(key, type, holderId); - return (processLock); + String key = entry.getKey(); + ProcessLockOrException processLockOrException = entry.getValue(); + if(processLockOrException.processLock() != null) + { + rs.put(key, processLockOrException); + stillNeedCreated.remove(key); + } + else if(processLockOrException.unableToObtainProcessLockException() != null) + { + lastExceptionsPerKey.put(key, processLockOrException.unableToObtainProcessLockException()); + } } - catch(UnableToObtainProcessLockException e) + + if(stillNeedCreated.isEmpty()) { - lastCaughtUnableToObtainProcessLockException = e; - if(Instant.now().plus(sleepBetweenTries).isBefore(giveUpTime)) - { - SleepUtils.sleep(sleepBetweenTries); - } - else - { - break; - } + ////////////////////////////////////////////////////////// + // if they've all been created now, great, return them! // + ////////////////////////////////////////////////////////// + return (rs); + } + + ///////////////////////////////////////////////////////////////////////////// + // oops, let's sleep (if we're before the give up time) and then try again // + ///////////////////////////////////////////////////////////////////////////// + if(Instant.now().plus(sleepBetweenTries).isBefore(giveUpTime)) + { + SleepUtils.sleep(sleepBetweenTries); + } + else + { + ///////////////////////////////// + // else, break if out of time! // + ///////////////////////////////// + break; } } - /////////////////////////////////////////////////////////////////////////////////////////////////// - // this variable can never be null with current code-path, but prefer to be defensive regardless // - /////////////////////////////////////////////////////////////////////////////////////////////////// - @SuppressWarnings("ConstantValue") - String suffix = lastCaughtUnableToObtainProcessLockException == null ? "" : ": " + lastCaughtUnableToObtainProcessLockException.getMessage(); + //////////////////////////////////////////////////////////////////////////////////////////// + // any that didn't get created, they need their last error (or a new error) put in the rs // + //////////////////////////////////////////////////////////////////////////////////////////// + for(String key : stillNeedCreated) + { + rs.put(key, new ProcessLockOrException(lastExceptionsPerKey.getOrDefault(key, new UnableToObtainProcessLockException("Missing key [" + key + "] in response from request to create lock. Lock not created.")))); + } - //noinspection ConstantValue - throw (new UnableToObtainProcessLockException("Unable to obtain process lock for key [" + key + "] in type [" + type + "] after [" + maxWait + "]" + suffix) - .withExistingLock(lastCaughtUnableToObtainProcessLockException == null ? null : lastCaughtUnableToObtainProcessLockException.getExistingLock())); + return (rs); } @@ -389,27 +640,41 @@ public class ProcessLockUtils { if(id == null) { - LOG.debug("No id passed in to releaseById - returning with noop"); + LOG.debug("No ids passed in to releaseById - returning with noop"); + return; + } + + releaseByIds(List.of(id)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void releaseByIds(List ids) + { + List nonNullIds = ids == null ? Collections.emptyList() : ids.stream().filter(Objects::nonNull).map(o -> (Serializable) o).toList(); + + if(CollectionUtils.nullSafeIsEmpty(nonNullIds)) + { + LOG.debug("No ids passed in to releaseById - returning with noop"); return; } - ProcessLock processLock = null; try { - processLock = ProcessLockUtils.getById(id); - if(processLock == null) + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKeys(nonNullIds)); + if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors())) { - LOG.info("Process lock not found in releaseById call", logPair("id", id)); + throw (new QException("Error deleting processLocks: " + deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString())); } + + LOG.info("Released process locks", logPair("ids", nonNullIds)); } catch(QException e) { - LOG.warn("Exception releasing processLock byId", e, logPair("id", id)); - } - - if(processLock != null) - { - release(processLock); + LOG.warn("Exception releasing processLocks byId", e, logPair("ids", ids)); } } @@ -420,26 +685,33 @@ public class ProcessLockUtils *******************************************************************************/ public static void release(ProcessLock processLock) { - try + if(processLock == null) { - if(processLock == null) - { - LOG.debug("No process lock passed in to release - returning with noop"); - return; - } - - DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKey(processLock.getId())); - if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors())) - { - throw (new QException("Error deleting processLock record: " + deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString())); - } - - LOG.info("Released process lock", logPair("id", processLock.getId()), logPair("key", processLock.getKey()), logPair("typeId", processLock.getProcessLockTypeId()), logPair("details", processLock.getDetails())); + LOG.debug("No process lock passed in to release - returning with noop"); + return; } - catch(QException e) + + releaseMany(List.of(processLock)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void releaseMany(List processLocks) + { + if(CollectionUtils.nullSafeIsEmpty(processLocks)) { - LOG.warn("Exception releasing processLock", e, logPair("processLockId", () -> processLock.getId())); + LOG.debug("No process locks passed in to release - returning with noop"); + return; } + + List ids = processLocks.stream() + .filter(Objects::nonNull) + .map(pl -> (Serializable) pl.getId()) + .toList(); + releaseByIds(ids); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/LoggingProcessTracer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/LoggingProcessTracer.java new file mode 100644 index 00000000..afaa80fe --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/LoggingProcessTracer.java @@ -0,0 +1,154 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.logging.LogPair; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Implementation of ProcessTracerInterface that writes messages to the Logger. + *******************************************************************************/ +public class LoggingProcessTracer implements ProcessTracerInterface +{ + private static final QLogger LOG = QLogger.getLogger(LoggingProcessTracer.class); + + private long startMillis; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleProcessStart(RunProcessInput runProcessInput) + { + startMillis = System.currentTimeMillis(); + LOG.info("Starting process", logPair("name", runProcessInput.getProcessName()), logPair("uuid", runProcessInput.getProcessUUID())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleProcessResume(RunProcessInput runProcessInput) + { + String atOrAfter = "after"; + String atOrAfterStep = runProcessInput.getStartAfterStep(); + if(StringUtils.hasContent(runProcessInput.getStartAtStep())) + { + atOrAfter = "at"; + atOrAfterStep = runProcessInput.getStartAtStep(); + } + LOG.info("Resuming process", logPair("name", runProcessInput.getProcessName()), logPair("uuid", runProcessInput.getProcessUUID()), logPair(atOrAfter, atOrAfterStep)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleStepStart(RunBackendStepInput runBackendStepInput) + { + LOG.info("Starting process step", runBackendStepInput.getStepName()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleMessage(RunBackendStepInput runBackendStepInput, ProcessTracerMessage message) + { + LOG.info("Message from process", logPair("message", message.getMessage())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleStepFinish(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + LOG.info("Finished process step", logPair("name", runBackendStepInput.getStepName())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleProcessBreak(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException) + { + LOG.info("Breaking process", logPair("name", runProcessInput.getProcessName()), logPair("uuid", runProcessInput.getProcessUUID())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleProcessFinish(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException) + { + long finishMillis = System.currentTimeMillis(); + + List summaryLogPairs = new ArrayList<>(); + Serializable processSummary = runProcessOutput.getValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + if(processSummary instanceof List) + { + List processSummaryLines = (List) processSummary; + for(ProcessSummaryLineInterface processSummaryLineInterface : processSummaryLines) + { + if(processSummaryLineInterface instanceof ProcessSummaryLine processSummaryLine) + { + summaryLogPairs.add(logPair(String.valueOf(summaryLogPairs.size()), logPair("status", processSummaryLine.getStatus()), logPair("count", processSummaryLine.getCount()), logPair("message", processSummaryLine.getMessage()))); + } + else + { + summaryLogPairs.add(logPair(String.valueOf(summaryLogPairs.size()), logPair("message", processSummaryLineInterface.getMessage()))); + } + } + } + + LOG.info("Finished process", logPair("name", runProcessInput.getProcessName()), logPair("uuid", runProcessInput.getProcessUUID()), logPair("millis", finishMillis - startMillis), logPair("summary", summaryLogPairs)); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/NoopProcessTracer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/NoopProcessTracer.java new file mode 100644 index 00000000..aeee851a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/NoopProcessTracer.java @@ -0,0 +1,107 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; + + +/******************************************************************************* + ** Implementation of ProcessTracerInterface that does nothing (no-op). + *******************************************************************************/ +public class NoopProcessTracer implements ProcessTracerInterface +{ + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleProcessStart(RunProcessInput runProcessInput) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleProcessResume(RunProcessInput runProcessInput) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleStepStart(RunBackendStepInput runBackendStepInput) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleMessage(RunBackendStepInput runBackendStepInput, ProcessTracerMessage message) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleStepFinish(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleProcessBreak(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException) + { + } + + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void handleProcessFinish(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException) + { + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerInterface.java new file mode 100644 index 00000000..e2d2ac84 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerInterface.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; + + +/******************************************************************************* + ** Interface that can be plugged into the execution of a QProcess, that gets + ** callbacks from QQQ for events in the lifecycle of a process, which one may + ** wish to log or otherwise be aware of. + *******************************************************************************/ +public interface ProcessTracerInterface +{ + /*************************************************************************** + ** Called when a new process is started. + ***************************************************************************/ + void handleProcessStart(RunProcessInput runProcessInput); + + /*************************************************************************** + ** Called when a process is resumed, e.g., after a "break" occurs between + ** backend steps and frontend steps. + ***************************************************************************/ + void handleProcessResume(RunProcessInput runProcessInput); + + /*************************************************************************** + ** Called when a (backend) step is started. + ***************************************************************************/ + void handleStepStart(RunBackendStepInput runBackendStepInput); + + /*************************************************************************** + ** Called when the (application, custom) process step code itself decides to + ** trace something. We imagine various subclasses of ProcessTracerMessage + ** to be created, to communicate more specific data for the tracer implementation. + ***************************************************************************/ + void handleMessage(RunBackendStepInput runBackendStepInput, ProcessTracerMessage message); + + /*************************************************************************** + ** Called when a (backend) step finishes. + ***************************************************************************/ + void handleStepFinish(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput); + + /*************************************************************************** + ** Called when a process break occurs, e.g., between backend and frontend + ** steps (but only if there are no more backend steps in the queue). + ***************************************************************************/ + void handleProcessBreak(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException); + + /*************************************************************************** + ** Called after the last (backend) step of a process. + ***************************************************************************/ + void handleProcessFinish(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java new file mode 100644 index 00000000..53938092 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java @@ -0,0 +1,66 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +/******************************************************************************* + ** Specialization of process tracer message, to indicate a 'key record' that was + ** used as an input or trigger to a process. + *******************************************************************************/ +public class ProcessTracerKeyRecordMessage extends ProcessTracerMessage +{ + private final String tableName; + private final Integer recordId; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public ProcessTracerKeyRecordMessage(String tableName, Integer recordId) + { + super("Process Key Record is " + tableName + " " + recordId); + this.tableName = tableName; + this.recordId = recordId; + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Getter for recordId + *******************************************************************************/ + public Integer getRecordId() + { + return (this.recordId); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java new file mode 100644 index 00000000..bb27a0e5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Basic class that can be passed in to ProcessTracerInterface.handleMessage. + ** This class just provides for a string message. We anticipate subclasses + ** that may have more specific data, that specific tracer implementations may + ** be aware of. + *******************************************************************************/ +public class ProcessTracerMessage implements Serializable +{ + private String message; + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ProcessTracerMessage() + { + } + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ProcessTracerMessage(String message) + { + this.message = message; + } + + + + /******************************************************************************* + ** Getter for message + *******************************************************************************/ + public String getMessage() + { + return (this.message); + } + + + + /******************************************************************************* + ** Setter for message + *******************************************************************************/ + public void setMessage(String message) + { + this.message = message; + } + + + + /******************************************************************************* + ** Fluent setter for message + *******************************************************************************/ + public ProcessTracerMessage withMessage(String message) + { + this.message = message; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java new file mode 100644 index 00000000..40099988 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java @@ -0,0 +1,343 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.text.ParseException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** class to give a human-friendly descriptive string from a cron expression. + ** (written in half by my friend Mr. Chatty G) + *******************************************************************************/ +public class CronDescriber +{ + private static final Map DAY_OF_WEEK_MAP = new HashMap<>(); + private static final Map MONTH_MAP = new HashMap<>(); + + static + { + DAY_OF_WEEK_MAP.put("1", "Sunday"); + DAY_OF_WEEK_MAP.put("2", "Monday"); + DAY_OF_WEEK_MAP.put("3", "Tuesday"); + DAY_OF_WEEK_MAP.put("4", "Wednesday"); + DAY_OF_WEEK_MAP.put("5", "Thursday"); + DAY_OF_WEEK_MAP.put("6", "Friday"); + DAY_OF_WEEK_MAP.put("7", "Saturday"); + + //////////////////////////////// + // Quartz also allows SUN-SAT // + //////////////////////////////// + DAY_OF_WEEK_MAP.put("SUN", "Sunday"); + DAY_OF_WEEK_MAP.put("MON", "Monday"); + DAY_OF_WEEK_MAP.put("TUE", "Tuesday"); + DAY_OF_WEEK_MAP.put("WED", "Wednesday"); + DAY_OF_WEEK_MAP.put("THU", "Thursday"); + DAY_OF_WEEK_MAP.put("FRI", "Friday"); + DAY_OF_WEEK_MAP.put("SAT", "Saturday"); + + MONTH_MAP.put("1", "January"); + MONTH_MAP.put("2", "February"); + MONTH_MAP.put("3", "March"); + MONTH_MAP.put("4", "April"); + MONTH_MAP.put("5", "May"); + MONTH_MAP.put("6", "June"); + MONTH_MAP.put("7", "July"); + MONTH_MAP.put("8", "August"); + MONTH_MAP.put("9", "September"); + MONTH_MAP.put("10", "October"); + MONTH_MAP.put("11", "November"); + MONTH_MAP.put("12", "December"); + + //////////////////////////////// + // Quartz also allows JAN-DEC // + //////////////////////////////// + MONTH_MAP.put("JAN", "January"); + MONTH_MAP.put("FEB", "February"); + MONTH_MAP.put("MAR", "March"); + MONTH_MAP.put("APR", "April"); + MONTH_MAP.put("MAY", "May"); + MONTH_MAP.put("JUN", "June"); + MONTH_MAP.put("JUL", "July"); + MONTH_MAP.put("AUG", "August"); + MONTH_MAP.put("SEP", "September"); + MONTH_MAP.put("OCT", "October"); + MONTH_MAP.put("NOV", "November"); + MONTH_MAP.put("DEC", "December"); + } + + /*************************************************************************** + ** + ***************************************************************************/ + public static String getDescription(String cronExpression) throws ParseException + { + String[] parts = cronExpression.trim().toUpperCase().split("\\s+"); + if(parts.length < 6 || parts.length > 7) + { + throw new ParseException("Invalid cron expression: " + cronExpression, 0); + } + + String seconds = parts[0]; + String minutes = parts[1]; + String hours = parts[2]; + String dayOfMonth = parts[3]; + String month = parts[4]; + String dayOfWeek = parts[5]; + String year = parts.length == 7 ? parts[6] : "*"; + + StringBuilder description = new StringBuilder(); + + description.append("At "); + description.append(describeTime(seconds, minutes, hours)); + description.append(", on "); + description.append(describeDayOfMonth(dayOfMonth)); + description.append(" of "); + description.append(describeMonth(month)); + description.append(", "); + description.append(describeDayOfWeek(dayOfWeek)); + if(!year.equals("*")) + { + description.append(", in ").append(year); + } + description.append("."); + + return description.toString(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeTime(String seconds, String minutes, String hours) + { + return String.format("%s, %s, %s", describePart(seconds, "second"), describePart(minutes, "minute"), describePart(hours, "hour")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeDayOfMonth(String dayOfMonth) + { + if(dayOfMonth.equals("?")) + { + return "every day"; + } + else if(dayOfMonth.equals("L")) + { + return "the last day"; + } + else if(dayOfMonth.contains("W")) + { + return "the nearest weekday to day " + dayOfMonth.replace("W", ""); + } + else + { + return (describePart(dayOfMonth, "day")); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeMonth(String month) + { + if(month.equals("*")) + { + return "every month"; + } + else if(month.contains("-")) + { + String[] parts = month.split("-"); + return String.format("%s to %s", MONTH_MAP.getOrDefault(parts[0], parts[0]), MONTH_MAP.getOrDefault(parts[1], parts[1])); + } + else + { + String[] months = month.split(","); + List monthNames = Arrays.stream(months).map(m -> MONTH_MAP.getOrDefault(m, m)).toList(); + return StringUtils.joinWithCommasAndAnd(monthNames); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeDayOfWeek(String dayOfWeek) + { + if(dayOfWeek.equals("?") || dayOfWeek.equals("*")) + { + return "every day of the week"; + } + else if(dayOfWeek.equals("L")) + { + return "the last day of the week"; + } + else if(dayOfWeek.contains("#")) + { + String[] parts = dayOfWeek.split("#"); + return String.format("the %s %s of the month", ordinal(parts[1]), DAY_OF_WEEK_MAP.getOrDefault(parts[0], parts[0])); + } + else if(dayOfWeek.contains("-")) + { + String[] parts = dayOfWeek.split("-"); + return String.format("from %s to %s", DAY_OF_WEEK_MAP.getOrDefault(parts[0], parts[0]), DAY_OF_WEEK_MAP.getOrDefault(parts[1], parts[1])); + } + else + { + String[] days = dayOfWeek.split(","); + List dayNames = Arrays.stream(days).map(d -> DAY_OF_WEEK_MAP.getOrDefault(d, d)).toList(); + return StringUtils.joinWithCommasAndAnd(dayNames); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describePart(String part, String label) + { + if(part.equals("*")) + { + return "every " + label; + } + else if(part.contains("/")) + { + String[] parts = part.split("/"); + if(parts[0].equals("*")) + { + parts[0] = "0"; + } + return String.format("every %s " + label + "s starting at %s", parts[1], parts[0]); + } + else if(part.contains(",")) + { + List partsList = Arrays.stream(part.split(",")).toList(); + + if(label.equals("hour")) + { + List hourNames = partsList.stream().map(p -> hourToAmPm(p)).toList(); + return StringUtils.joinWithCommasAndAnd(hourNames); + } + else + { + if(label.equals("day")) + { + return "days " + StringUtils.joinWithCommasAndAnd(partsList); + } + else + { + return StringUtils.joinWithCommasAndAnd(partsList) + " " + label + "s"; + } + } + } + else if(part.contains("-")) + { + String[] parts = part.split("-"); + if(label.equals("day")) + { + return String.format("%ss from %s to %s", label, parts[0], parts[1]); + } + else if(label.equals("hour")) + { + return String.format("from %s to %s", hourToAmPm(parts[0]), hourToAmPm(parts[1])); + } + else + { + return String.format("from %s to %s %s", parts[0], parts[1], label + "s"); + } + } + else + { + if(label.equals("day")) + { + return label + " " + part; + } + if(label.equals("hour")) + { + return hourToAmPm(part); + } + else + { + return part + " " + label + "s"; + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String hourToAmPm(String part) + { + try + { + int hour = Integer.parseInt(part); + return switch(hour) + { + case 0 -> "midnight"; + case 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 -> hour + " AM"; + case 12 -> "noon"; + case 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 -> (hour - 12) + " PM"; + default -> hour + " hours"; + }; + } + catch(Exception e) + { + return part + " hours"; + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String ordinal(String number) + { + int n = Integer.parseInt(number); + if(n >= 11 && n <= 13) + { + return n + "th"; + } + + return switch(n % 10) + { + case 1 -> n + "st"; + case 2 -> n + "nd"; + case 3 -> n + "rd"; + default -> n + "th"; + }; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java new file mode 100644 index 00000000..bd77f6b4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Field display behavior, to add a human-redable tooltip to cron-expressions. + *******************************************************************************/ +public class CronExpressionTooltipFieldBehavior implements FieldDisplayBehavior +{ + + /*************************************************************************** + ** Add both this behavior, and the tooltip adornment to a field + ** Note, if either was already there, then that part is left alone. + ***************************************************************************/ + public static void addToField(QFieldMetaData fieldMetaData) + { + CronExpressionTooltipFieldBehavior existingBehavior = fieldMetaData.getBehaviorOnlyIfSet(CronExpressionTooltipFieldBehavior.class); + if(existingBehavior == null) + { + fieldMetaData.withBehavior(new CronExpressionTooltipFieldBehavior()); + } + + if(fieldMetaData.getAdornment(AdornmentType.TOOLTIP).isEmpty()) + { + fieldMetaData.withFieldAdornment((new FieldAdornment(AdornmentType.TOOLTIP) + .withValue(AdornmentType.TooltipValues.TOOLTIP_DYNAMIC, true))); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : recordList) + { + try + { + String cronExpression = record.getValueString(field.getName()); + if(StringUtils.hasContent(cronExpression)) + { + String description = CronDescriber.getDescription(cronExpression); + record.setDisplayValue(field.getName() + ":" + AdornmentType.TooltipValues.TOOLTIP_DYNAMIC, description); + } + } + catch(Exception e) + { + ///////////////////// + // just leave null // + ///////////////////// + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 3dfec3ae..88246600 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -441,11 +441,16 @@ public class QScheduleManager try { HashMap parameters = new HashMap<>(paramMap); - HashMap variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + String variantOptionsTableName = backendMetaData.getBackendVariantsConfig().getOptionsTableName(); + String variantOptionsTableIdFieldName = QContext.getQInstance().getTable(variantOptionsTableName).getPrimaryKeyField(); + + HashMap variantMap = new HashMap<>(Map.of(variantTypeKey, qRecord.getValue(variantOptionsTableIdFieldName))); parameters.put("backendVariantData", variantMap); - String identity = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); - String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + String identity = schedulableIdentity.getIdentity() + ";" + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName); + String description = schedulableIdentity.getDescription() + " for variant: " + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName); BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java index c7cc6a4c..e7b46ea3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -34,9 +34,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -102,7 +99,8 @@ public class SchedulerUtils try { QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); - Map thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())); + QTableMetaData variantTable = QContext.getQInstance().getTable(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + Map thisVariantData = MapBuilder.of(backendMetaData.getBackendVariantsConfig().getVariantTypeKey(), qRecord.getValue(variantTable.getPrimaryKeyField())); executeSingleProcess(process, thisVariantData, processInputValues); } catch(Exception e) @@ -181,8 +179,8 @@ public class SchedulerUtils QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(processMetaData.getVariantBackend()); QueryInput queryInput = new QueryInput(); - queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); + queryInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + queryInput.setFilter(backendMetaData.getBackendVariantsConfig().getOptionsFilter()); QueryOutput queryOutput = new QueryAction().execute(queryInput); records = queryOutput.getRecords(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java index 84f893fb..4d9f81e5 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java @@ -83,7 +83,7 @@ public class StringUtils /******************************************************************************* - ** allCapsToMixedCase - ie, UNIT CODE -> Unit Code + ** allCapsToMixedCase - ie, UNIT_CODE -> Unit Code ** ** @param input ** @return @@ -128,7 +128,7 @@ public class StringUtils return (input); } - return (rs.toString()); + return (rs.toString().replace('_', ' ')); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index f2ee6802..1e791abd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -590,11 +590,35 @@ public class ValueUtils try { + ///////////////////////////////////////////////////////////////////////////////////// + // first assume the instant is perfectly formatted, as in: 2007-12-03T10:15:30.00Z // + ///////////////////////////////////////////////////////////////////////////////////// return Instant.parse(s); } catch(DateTimeParseException e) { - return tryAlternativeInstantParsing(s, e); + try + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the string isn't quite the right format, try some alternates that are common and fairly un-vague // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + return tryAlternativeInstantParsing(s, e); + } + catch(DateTimeParseException dtpe) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // we commonly receive date-times with only a single-digit hour after the space, which fails tryAlternativeInstantParsing. // + // so if we see what looks like that pattern, zero-pad the hour, and try the alternative parse patterns again. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(s.matches(".* \\d:.*")) + { + return tryAlternativeInstantParsing(s.replaceFirst(" (\\d):", " 0$1:"), e); + } + else + { + throw (dtpe); + } + } } } else @@ -617,11 +641,12 @@ public class ValueUtils /******************************************************************************* ** *******************************************************************************/ - private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e) + private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e) throws DateTimeParseException { - ////////////////////// - // 1999-12-31T12:59 // - ////////////////////// + //////////////////////////////////////////////////////////////////// + // 1999-12-31T12:59 // + // missing seconds & zone - but we're happy to assume :00 and UTC // + //////////////////////////////////////////////////////////////////// if(s.matches("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}$")) { ////////////////////////// @@ -630,27 +655,30 @@ public class ValueUtils return Instant.parse(s + ":00Z"); } - /////////////////////////// - // 1999-12-31 12:59:59.0 // - /////////////////////////// + /////////////////////////////////////////////////////////////// + // 1999-12-31 12:59:59.0 // + // fractional seconds and no zone - truncate, and assume UTC // + /////////////////////////////////////////////////////////////// else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.0$")) { s = s.replaceAll(" ", "T").replaceAll("\\..*$", "Z"); return Instant.parse(s); } - ///////////////////////// - // 1999-12-31 12:59:59 // - ///////////////////////// + //////////////////////////////////////////// + // 1999-12-31 12:59:59 // + // Missing 'T' and 'Z', so just add those // + //////////////////////////////////////////// else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$")) { s = s.replaceAll(" ", "T") + "Z"; return Instant.parse(s); } - ////////////////////// - // 1999-12-31 12:59 // - ////////////////////// + ///////////////////////////////////////////// + // 1999-12-31 12:59 // + // missing T, seconds, and Z - add 'em all // + ///////////////////////////////////////////// else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$")) { s = s.replaceAll(" ", "T") + ":00Z"; @@ -661,12 +689,18 @@ public class ValueUtils { try { + //////////////////////////////////////////////////////// + // such as '2011-12-03T10:15:30+01:00[Europe/Paris]'. // + //////////////////////////////////////////////////////// return LocalDateTime.parse(s, DateTimeFormatter.ISO_ZONED_DATE_TIME).toInstant(ZoneOffset.UTC); } catch(DateTimeParseException e2) { try { + /////////////////////////////////////////////////////// + // also includes such as '2011-12-03T10:15:30+01:00' // + /////////////////////////////////////////////////////// return LocalDateTime.parse(s, DateTimeFormatter.ISO_DATE_TIME).toInstant(ZoneOffset.UTC); } catch(Exception e3) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java new file mode 100644 index 00000000..8cbcb54d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java @@ -0,0 +1,53 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Map; +import java.util.function.Supplier; + + +/******************************************************************************* + ** Version of map where string keys are handled case-insensitively. e.g., + ** map.put("One", 1); map.get("ONE") == 1. + *******************************************************************************/ +public class CaseInsensitiveKeyMap extends TransformedKeyMap +{ + /*************************************************************************** + * + ***************************************************************************/ + public CaseInsensitiveKeyMap() + { + super(key -> key.toLowerCase()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public CaseInsensitiveKeyMap(Supplier> supplier) + { + super(key -> key.toLowerCase(), supplier); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java new file mode 100644 index 00000000..ae424e66 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java @@ -0,0 +1,401 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + +/******************************************************************************* + ** Version of a map that uses a transformation function on keys. The original + ** idea being, e.g., to support case-insensitive keys via a toLowerCase transform. + ** e.g., map.put("One", 1); map.get("ONE") == 1. + ** + ** But, implemented generically to support any transformation function. + ** + ** keySet() and entries() should give only the first version of a key that overlapped. + ** e.g., map.put("One", 1); map.put("one", 1); map.keySet() == Set.of("One"); + *******************************************************************************/ +public class TransformedKeyMap implements Map +{ + private Function keyTransformer; + private Map wrappedMap; + + private Map originalKeys = new HashMap<>(); + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMap(Function keyTransformer) + { + this.keyTransformer = keyTransformer; + this.wrappedMap = new HashMap<>(); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMap(Function keyTransformer, Supplier> supplier) + { + this.keyTransformer = keyTransformer; + this.wrappedMap = supplier.get(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public int size() + { + return (wrappedMap.size()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean isEmpty() + { + return (wrappedMap.isEmpty()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean containsKey(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + return wrappedMap.containsKey(transformed); + } + catch(Exception e) + { + return (false); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean containsValue(Object value) + { + return (wrappedMap.containsValue(value)); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public V get(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + return wrappedMap.get(transformed); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @Nullable V put(OK key, V value) + { + TK transformed = keyTransformer.apply(key); + originalKeys.putIfAbsent(transformed, key); + return wrappedMap.put(transformed, value); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public V remove(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + originalKeys.remove(transformed); + return wrappedMap.remove(transformed); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void putAll(@NotNull Map m) + { + for(Entry entry : m.entrySet()) + { + put(entry.getKey(), entry.getValue()); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void clear() + { + wrappedMap.clear(); + originalKeys.clear(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Set keySet() + { + return new HashSet<>(originalKeys.values()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Collection values() + { + return wrappedMap.values(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Set> entrySet() + { + Set> wrappedEntries = wrappedMap.entrySet(); + Set> originalEntries; + try + { + originalEntries = wrappedEntries.getClass().getConstructor().newInstance(); + } + catch(Exception e) + { + originalEntries = new HashSet<>(); + } + + for(Entry wrappedEntry : wrappedEntries) + { + OK originalKey = originalKeys.get(wrappedEntry.getKey()); + originalEntries.add(new TransformedKeyMapEntry<>(originalKey, wrappedEntry.getValue())); + } + + return (originalEntries); + } + + // methods with a default implementation below here // + + + + /* + @Override + public V getOrDefault(Object key, V defaultValue) + { + return Map.super.getOrDefault(key, defaultValue); + } + + + + @Override + public void forEach(BiConsumer action) + { + Map.super.forEach(action); + } + + + + @Override + public void replaceAll(BiFunction function) + { + Map.super.replaceAll(function); + } + + + + @Override + public @Nullable V putIfAbsent(OK key, V value) + { + return Map.super.putIfAbsent(key, value); + } + + + + @Override + public boolean remove(Object key, Object value) + { + return Map.super.remove(key, value); + } + + + + @Override + public boolean replace(OK key, V oldValue, V newValue) + { + return Map.super.replace(key, oldValue, newValue); + } + + + + @Override + public @Nullable V replace(OK key, V value) + { + return Map.super.replace(key, value); + } + + + + @Override + public V computeIfAbsent(OK key, @NotNull Function mappingFunction) + { + return Map.super.computeIfAbsent(key, mappingFunction); + } + + + + @Override + public V computeIfPresent(OK key, @NotNull BiFunction remappingFunction) + { + return Map.super.computeIfPresent(key, remappingFunction); + } + + + + @Override + public V compute(OK key, @NotNull BiFunction remappingFunction) + { + return Map.super.compute(key, remappingFunction); + } + + + + @Override + public V merge(OK key, @NotNull V value, @NotNull BiFunction remappingFunction) + { + return Map.super.merge(key, value, remappingFunction); + } + */ + + + + /*************************************************************************** + * + ***************************************************************************/ + public static class TransformedKeyMapEntry implements Map.Entry + { + private final EK key; + private EV value; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMapEntry(EK key, EV value) + { + this.key = key; + this.value = value; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EK getKey() + { + return (key); + } + + + + @Override + public EV getValue() + { + return (value); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public EV setValue(EV value) + { + throw (new UnsupportedOperationException("Setting value in an entry of a TransformedKeyMap is not supported.")); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java index 4ed6cbce..ac02e21b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java @@ -247,6 +247,16 @@ public class Memoization + /******************************************************************************* + ** + *******************************************************************************/ + public void clearKey(K key) + { + this.map.remove(key); + } + + + /******************************************************************************* ** Setter for timeoutSeconds ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterfaceTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterfaceTest.java new file mode 100644 index 00000000..c4129707 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterfaceTest.java @@ -0,0 +1,325 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for TableCustomizerInterface + *******************************************************************************/ +class TableCustomizerInterfaceTest extends BaseTest +{ + private static List events = new ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + events.clear(); + QLogger.deactivateCollectingLoggerForClass(TableCustomizerInterface.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreInsertOnly() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PreInsertOnly.class)) + .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(PreInsertOnly.class)); + reInitInstanceInContext(qInstance); + + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(TableCustomizerInterface.class); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1).contains("PreInsertOnly.preInsert()"); + assertThat(collectingLogger.getCollectedMessages()).hasSize(0); + + new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1); + assertThat(collectingLogger.getCollectedMessages()).hasSize(1).element(0).extracting("message").asString().contains("A default implementation of preUpdate is running"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreUpdateOnly() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(PreUpdateOnly.class)) + .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PreUpdateOnly.class)); + reInitInstanceInContext(qInstance); + + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(TableCustomizerInterface.class); + + new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1).contains("PreUpdateOnly.preUpdate()"); + assertThat(collectingLogger.getCollectedMessages()).hasSize(0); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1); + assertThat(collectingLogger.getCollectedMessages()).hasSize(1).element(0).extracting("message").asString().contains("A default implementation of preInsert is running"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreInsertOrUpdateOnly() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(PreInsertOrUpdateOnly.class)) + .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PreInsertOrUpdateOnly.class)); + reInitInstanceInContext(qInstance); + + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(TableCustomizerInterface.class); + + new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1).contains("PreInsertOrUpdateOnly.preInsertOrUpdate()"); + assertThat(collectingLogger.getCollectedMessages()).hasSize(0); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(2).allMatch(s -> s.contains("PreInsertOrUpdateOnly.preInsertOrUpdate()")); + assertThat(collectingLogger.getCollectedMessages()).hasSize(0); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostInsertOnly() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PostInsertOnly.class)) + .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(PostInsertOnly.class)); + reInitInstanceInContext(qInstance); + + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(TableCustomizerInterface.class); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1).contains("PostInsertOnly.postInsert()"); + assertThat(collectingLogger.getCollectedMessages()).hasSize(0); + + new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1); + assertThat(collectingLogger.getCollectedMessages()).hasSize(1).element(0).extracting("message").asString().contains("A default implementation of postUpdate is running"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostUpdateOnly() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(PostUpdateOnly.class)) + .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PostUpdateOnly.class)); + reInitInstanceInContext(qInstance); + + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(TableCustomizerInterface.class); + + new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1).contains("PostUpdateOnly.postUpdate()"); + assertThat(collectingLogger.getCollectedMessages()).hasSize(0); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1); + assertThat(collectingLogger.getCollectedMessages()).hasSize(1).element(0).extracting("message").asString().contains("A default implementation of postInsert is running"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostInsertOrUpdateOnly() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(PostInsertOrUpdateOnly.class)) + .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PostInsertOrUpdateOnly.class)); + reInitInstanceInContext(qInstance); + + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(TableCustomizerInterface.class); + + new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(1).contains("PostInsertOrUpdateOnly.postInsertOrUpdate()"); + assertThat(collectingLogger.getCollectedMessages()).hasSize(0); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord())); + assertThat(events).hasSize(2).allMatch(s -> s.contains("PostInsertOrUpdateOnly.postInsertOrUpdate()")); + assertThat(collectingLogger.getCollectedMessages()).hasSize(0); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class PreInsertOnly implements TableCustomizerInterface + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + events.add("PreInsertOnly.preInsert()"); + return (records); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class PreUpdateOnly implements TableCustomizerInterface + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + events.add("PreUpdateOnly.preUpdate()"); + return (records); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class PreInsertOrUpdateOnly implements TableCustomizerInterface + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List preInsertOrUpdate(AbstractActionInput input, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + events.add("PreInsertOrUpdateOnly.preInsertOrUpdate()"); + return (records); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class PostInsertOnly implements TableCustomizerInterface + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + events.add("PostInsertOnly.postInsert()"); + return (records); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class PostUpdateOnly implements TableCustomizerInterface + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + events.add("PostUpdateOnly.postUpdate()"); + return (records); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class PostInsertOrUpdateOnly implements TableCustomizerInterface + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List postInsertOrUpdate(AbstractActionInput input, List records, Optional> oldRecordList) throws QException + { + events.add("PostInsertOrUpdateOnly.postInsertOrUpdate()"); + return (records); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java new file mode 100644 index 00000000..64246384 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for RecordListWidgetRenderer + *******************************************************************************/ +class RecordListWidgetRendererTest extends BaseTest +{ + + /*************************************************************************** + ** + ***************************************************************************/ + private QWidgetMetaData defineWidget() + { + return RecordListWidgetRenderer.widgetMetaDataBuilder("testRecordListWidget") + .withTableName(TestUtils.TABLE_NAME_SHAPE) + .withMaxRows(20) + .withLabel("Some Shapes") + .withFilter(new QQueryFilter() + .withCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, "${input.maxShapeId}") + .withCriteria("name", QCriteriaOperator.NOT_EQUALS, "Square") + .withOrderBy(new QFilterOrderBy("id", false)) + ).getWidgetMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidation() throws QInstanceValidationException + { + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("tableName"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for tableName must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("filter"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for filter must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("tableName"); + widgetMetaData.getDefaultValues().remove("filter"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for filter must be given") + .hasMessageContaining("defaultValue for tableName must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + QQueryFilter filter = (QQueryFilter) widgetMetaData.getDefaultValues().get("filter"); + filter.addCriteria(new QFilterCriteria("noField", QCriteriaOperator.EQUALS, "noValue")); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("Criteria fieldName noField is not a field in this table"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + qInstance.addWidget(widgetMetaData); + + ////////////////////////////////// + // make sure valid setup passes // + ////////////////////////////////// + new QInstanceValidator().validate(qInstance); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRender() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + qInstance.addWidget(widgetMetaData); + + TestUtils.insertDefaultShapes(qInstance); + TestUtils.insertExtraShapes(qInstance); + + { + RecordListWidgetRenderer recordListWidgetRenderer = new RecordListWidgetRenderer(); + RenderWidgetInput input = new RenderWidgetInput(); + input.setWidgetMetaData(widgetMetaData); + input.setQueryParams(Map.of("maxShapeId", "1")); + RenderWidgetOutput output = recordListWidgetRenderer.render(input); + + ChildRecordListData widgetData = (ChildRecordListData) output.getWidgetData(); + assertEquals(1, widgetData.getTotalRows()); + assertEquals(1, widgetData.getQueryOutput().getRecords().get(0).getValue("id")); + assertEquals("Triangle", widgetData.getQueryOutput().getRecords().get(0).getValue("name")); + } + + { + RecordListWidgetRenderer recordListWidgetRenderer = new RecordListWidgetRenderer(); + RenderWidgetInput input = new RenderWidgetInput(); + input.setWidgetMetaData(widgetMetaData); + input.setQueryParams(Map.of("maxShapeId", "4")); + RenderWidgetOutput output = recordListWidgetRenderer.render(input); + + ChildRecordListData widgetData = (ChildRecordListData) output.getWidgetData(); + assertEquals(3, widgetData.getTotalRows()); + + ///////////////////////////////////////////////////////////////////////// + // id=2,name=Square was skipped due to NOT_EQUALS Square in the filter // + // max-shape-id applied we don't get id=5 or 6 // + // and they're ordered as specified in the filter (id desc) // + ///////////////////////////////////////////////////////////////////////// + assertEquals(4, widgetData.getQueryOutput().getRecords().get(0).getValue("id")); + assertEquals("Rectangle", widgetData.getQueryOutput().getRecords().get(0).getValue("name")); + + assertEquals(3, widgetData.getQueryOutput().getRecords().get(1).getValue("id")); + assertEquals("Circle", widgetData.getQueryOutput().getRecords().get(1).getValue("name")); + + assertEquals(1, widgetData.getQueryOutput().getRecords().get(2).getValue("id")); + assertEquals("Triangle", widgetData.getQueryOutput().getRecords().get(2).getValue("name")); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelperTest.java index cb61d4f7..eb71e15e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelperTest.java @@ -36,8 +36,10 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -523,6 +525,10 @@ class PermissionsHelperTest extends BaseTest { QInstance qInstance = new QInstance(); + qInstance.setAuthentication(new QAuthenticationMetaData() + .withType(QAuthenticationType.FULLY_ANONYMOUS) + .withName("anonymous")); + qInstance.addBackend(new QBackendMetaData() .withName("backend")); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java index 1d989f96..bf7eeb0a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java @@ -23,10 +23,15 @@ package com.kingsrook.qqq.backend.core.actions.processes; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceLambda; @@ -35,6 +40,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.utils.collections.MultiLevelMapHelper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -73,14 +80,14 @@ class RunProcessActionTest extends BaseTest ///////////////////////////////////////////////////////////////// // two-steps - a, points at b; b has no next-step, so it exits // ///////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") + .withStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepB"); @@ -109,8 +116,8 @@ class RunProcessActionTest extends BaseTest { QProcessMetaData process = new QProcessMetaData().withName("test") - .addStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b")) - .addStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend"))) + .withStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b")) + .withStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend"))) .withStepFlow(ProcessStepFlow.STATE_MACHINE); @@ -150,7 +157,7 @@ class RunProcessActionTest extends BaseTest // since it never goes to the frontend, it'll stack overflow // // (though we'll catch it ourselves before JVM does) // /////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); @@ -193,14 +200,14 @@ class RunProcessActionTest extends BaseTest // since it never goes to the frontend, it'll stack overflow // // (though we'll catch it ourselves before JVM does) // /////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") + .withStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepB"); @@ -238,7 +245,7 @@ class RunProcessActionTest extends BaseTest { QProcessMetaData process = new QProcessMetaData().withName("test") - .addStep(QStateMachineStep.frontendThenBackend("a", + .withStep(QStateMachineStep.frontendThenBackend("a", new QFrontendStepMetaData().withName("aFrontend"), new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -247,7 +254,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.frontendThenBackend("b", + .withStep(QStateMachineStep.frontendThenBackend("b", new QFrontendStepMetaData().withName("bFrontend"), new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -256,7 +263,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("c"); })))) - .addStep(QStateMachineStep.frontendThenBackend("c", + .withStep(QStateMachineStep.frontendThenBackend("c", new QFrontendStepMetaData().withName("cFrontend"), new QBackendStepMetaData().withName("cBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -265,7 +272,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("d"); })))) - .addStep(QStateMachineStep.frontendOnly("d", + .withStep(QStateMachineStep.frontendOnly("d", new QFrontendStepMetaData().withName("dFrontend"))) .withStepFlow(ProcessStepFlow.STATE_MACHINE); @@ -321,7 +328,132 @@ class RunProcessActionTest extends BaseTest runProcessOutput = new RunProcessAction().execute(input); assertEquals(List.of("in StepA", "in StepB", "in StepC"), log); assertThat(runProcessOutput.getProcessState().getNextStepName()).isEmpty(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGoingBack() throws QException + { + AtomicInteger backCount = new AtomicInteger(0); + Map stepRunCounts = new HashMap<>(); + + BackendStep backendStep = (runBackendStepInput, runBackendStepOutput) -> + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // shared backend-step lambda, that will do the same thing for both - but using step name to count how many times each is executed. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + MultiLevelMapHelper.getOrPutAndIncrement(stepRunCounts, runBackendStepInput.getStepName()); + if(runBackendStepInput.getIsStepBack()) + { + backCount.incrementAndGet(); + } + }; + + /////////////////////////////////////////////////////////// + // normal flow here: a -> b -> c // + // but, b can go back to a, as in: a -> b -> a -> b -> c // + /////////////////////////////////////////////////////////// + QProcessMetaData process = new QProcessMetaData().withName("test") + .withStep(new QBackendStepMetaData() + .withName("a") + .withCode(new QCodeReferenceLambda<>(backendStep))) + .withStep(new QFrontendStepMetaData() + .withName("b") + .withBackStepName("a")) + .withStep(new QBackendStepMetaData() + .withName("c") + .withCode(new QCodeReferenceLambda<>(backendStep))) + .withStepFlow(ProcessStepFlow.LINEAR); + + QContext.getQInstance().addProcess(process); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("test"); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + /////////////////////////////////////////////////////////// + // start the process - we should be sent to b (frontend) // + /////////////////////////////////////////////////////////// + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("b"); + + assertEquals(0, backCount.get()); + assertEquals(Map.of("a", 1), stepRunCounts); + + //////////////////////////////////////////////////////////////// + // resume after b, but in back-mode - should end up back at b // + //////////////////////////////////////////////////////////////// + input.setStartAfterStep(null); + input.setStartAtStep("a"); + runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("b"); + + assertEquals(1, backCount.get()); + assertEquals(Map.of("a", 2), stepRunCounts); + + //////////////////////////////////////////////////////////////////////////// + // resume after b, in regular (forward) mode - should wrap up the process // + //////////////////////////////////////////////////////////////////////////// + input.setStartAfterStep("b"); + input.setStartAtStep(null); + runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isEmpty(); + + assertEquals(1, backCount.get()); + assertEquals(Map.of("a", 2, "c", 1), stepRunCounts); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetAvailableStepList() throws QException + { + QProcessMetaData process = new QProcessMetaData() + .withStep(new QBackendStepMetaData().withName("A")) + .withStep(new QBackendStepMetaData().withName("B")) + .withStep(new QBackendStepMetaData().withName("C")) + .withStep(new QBackendStepMetaData().withName("D")) + .withStep(new QBackendStepMetaData().withName("E")); + + ProcessState processState = new ProcessState(); + processState.setStepList(process.getStepList().stream().map(s -> s.getName()).toList()); + + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, null, false)); + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, null, true)); + + assertStepListNames(List.of("B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "A", false)); + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "A", true)); + + assertStepListNames(List.of("D", "E"), RunProcessAction.getAvailableStepList(processState, process, "C", false)); + assertStepListNames(List.of("C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "C", true)); + + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "E", false)); + assertStepListNames(List.of("E"), RunProcessAction.getAvailableStepList(processState, process, "E", true)); + + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "Z", false)); + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "Z", true)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertStepListNames(List expectedNames, List actualSteps) + { + assertEquals(expectedNames, actualSteps.stream().map(s -> s.getName()).toList()); } } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java index 03db7e79..c2594727 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java @@ -90,7 +90,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; *******************************************************************************/ public class GenerateReportActionTest extends BaseTest { - private static final String REPORT_NAME = "personReport1"; + public static final String REPORT_NAME = "personReport1"; @@ -655,7 +655,7 @@ public class GenerateReportActionTest extends BaseTest Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(5, list.size()); - assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name"); + assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name", "Birth Date"); } @@ -663,7 +663,7 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private static QReportMetaData defineTableOnlyReport() + public static QReportMetaData defineTableOnlyReport() { QReportMetaData report = new QReportMetaData() .withName(REPORT_NAME) @@ -686,7 +686,9 @@ public class GenerateReportActionTest extends BaseTest .withColumns(List.of( new QReportField().withName("id"), new QReportField().withName("firstName"), - new QReportField().withName("lastName"))))); + new QReportField().withName("lastName"), + new QReportField().withName("birthDate") + )))); return report; } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java index ba1c7c9b..5a93f970 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java @@ -23,10 +23,14 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -50,4 +54,18 @@ class CountActionTest extends BaseTest CountOutput result = new CountAction().execute(request); assertNotNull(result); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStaticWrapper() throws QException + { + TestUtils.insertDefaultShapes(QContext.getQInstance()); + assertEquals(3, CountAction.execute(TestUtils.TABLE_NAME_SHAPE, null)); + assertEquals(3, CountAction.execute(TestUtils.TABLE_NAME_SHAPE, new QQueryFilter())); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateActionTest.java index 1f448b69..33c36769 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateActionTest.java @@ -66,16 +66,14 @@ public class RenderTemplateActionTest extends BaseTest @Test void testConvenientWrapper() throws QException { - RenderTemplateInput parentActionInput = new RenderTemplateInput(); - String template = "Hello, $name"; - assertEquals("Hello, Darin", RenderTemplateAction.renderVelocity(parentActionInput, Map.of("name", "Darin"), template)); - assertEquals("Hello, Tim", RenderTemplateAction.renderVelocity(parentActionInput, Map.of("name", "Tim"), template)); - assertEquals("Hello, $name", RenderTemplateAction.renderVelocity(parentActionInput, Map.of(), template)); + assertEquals("Hello, Darin", RenderTemplateAction.renderVelocity(Map.of("name", "Darin"), template)); + assertEquals("Hello, Tim", RenderTemplateAction.renderVelocity(Map.of("name", "Tim"), template)); + assertEquals("Hello, $name", RenderTemplateAction.renderVelocity(Map.of(), template)); template = "Hello, $!name"; - assertEquals("Hello, ", RenderTemplateAction.renderVelocity(parentActionInput, Map.of(), template)); - assertEquals("Hello, ", RenderTemplateAction.renderVelocity(parentActionInput, null, template)); + assertEquals("Hello, ", RenderTemplateAction.renderVelocity(Map.of(), template)); + assertEquals("Hello, ", RenderTemplateAction.renderVelocity(null, template)); } @@ -128,4 +126,4 @@ public class RenderTemplateActionTest extends BaseTest throw (new Exception("You asked to throw...")); } -} \ No newline at end of file +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java index cd678974..05d24862 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java @@ -30,19 +30,23 @@ import java.time.LocalTime; import java.time.Month; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -237,4 +241,102 @@ class QValueFormatterTest extends BaseTest assertEquals("2024-04-04 02:12:00 PM CDT", record.getDisplayValue("createDate")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBlobValuesToDownloadUrls() + { + byte[] blobBytes = "hello".getBytes(); + { + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB) + .withFieldAdornment(new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "blob-%s.txt") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS, new ArrayList<>(List.of("id"))))); + + ////////////////////////////////////////////////////////////////// + // verify display value gets set to formated file-name + fields // + // and raw value becomes URL for downloading the byte // + ////////////////////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/blob-47.txt", record.getValueString("blobField")); + assertEquals("blob-47.txt", record.getDisplayValue("blobField")); + + //////////////////////////////////////////////////////// + // verify that w/ no blob value, we don't do anything // + //////////////////////////////////////////////////////// + QRecord recordWithoutBlobValue = new QRecord().withValue("id", 47); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(recordWithoutBlobValue)); + assertNull(recordWithoutBlobValue.getValue("blobField")); + assertNull(recordWithoutBlobValue.getDisplayValue("blobField")); + } + + { + FieldAdornment adornment = new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName"); + + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB) + .withFieldAdornment(adornment)); + + //////////////////////////////////////////////////// + // here get the file name directly from one field // + //////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes).withValue("fileName", "myBlob.txt"); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/myBlob.txt", record.getValueString("blobField")); + assertEquals("myBlob.txt", record.getDisplayValue("blobField")); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // switch to use dynamic url, rerun, and assert we get the values as they were on the record before the call // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + adornment.withValue(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC, true); + record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes).withValue("fileName", "myBlob.txt") + .withDisplayValue("blobField:" + AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC, "/something-custom/") + .withDisplayValue("blobField", "myDisplayValue"); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertArrayEquals(blobBytes, record.getValueByteArray("blobField")); + assertEquals("myDisplayValue", record.getDisplayValue("blobField")); + } + + { + FieldAdornment adornment = new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD); + + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withLabel("Test Table") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB).withLabel("Blob").withFieldAdornment(adornment)); + + /////////////////////////////////////////////////////////////////////////////////////////// + // w/o file name format or whatever, generate a file name from table & id & field labels // + /////////////////////////////////////////////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/Test%20Table%2047%20Blob", record.getValueString("blobField")); + assertEquals("Test Table 47 Blob", record.getDisplayValue("blobField")); + + //////////////////////////////////////// + // add a default extension and re-run // + //////////////////////////////////////// + adornment.withValue(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION, "html"); + record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/Test%20Table%2047%20Blob.html", record.getValueString("blobField")); + assertEquals("Test Table 47 Blob.html", record.getDisplayValue("blobField")); + } + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java index ee2bb104..d71f6335 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java @@ -217,6 +217,63 @@ class SearchPossibleValueSourceActionTest extends BaseTest } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPvsAction_tableByLabels() throws QException + { + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("Square", "Circle"), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("Square")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(3) && pv.getLabel().equals("Circle")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of(), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(0, output.getResults().size()); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("notFound"), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(0, output.getResults().size()); + } + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPvsAction_enumByLabel() throws QException + { + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("IL", "MO", "XX"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1) && pv.getLabel().equals("IL")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("MO")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("Il", "mo", "XX"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1) && pv.getLabel().equals("IL")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("MO")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of(), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(0, output.getResults().size()); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("not-found"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(0, output.getResults().size()); + } + } + + /******************************************************************************* ** @@ -414,4 +471,18 @@ class SearchPossibleValueSourceActionTest extends BaseTest return output; } + + + /******************************************************************************* + ** + *******************************************************************************/ + private SearchPossibleValueSourceOutput getSearchPossibleValueSourceOutputByLabels(List labels, String possibleValueSourceName) throws QException + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); + input.setLabelList(labels); + input.setPossibleValueSourceName(possibleValueSourceName); + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input); + return output; + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/context/QContextTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/context/QContextTest.java index 78c96db4..7bd44fdf 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/context/QContextTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/context/QContextTest.java @@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.core.context; import java.util.Map; import java.util.UUID; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -102,6 +104,7 @@ class QContextTest private static QInstance newQInstance() { QInstance qInstance = new QInstance(); + qInstance.setAuthentication(new QAuthenticationMetaData().withType(QAuthenticationType.FULLY_ANONYMOUS).withName("anonymous")); qInstance.addBackend(new QBackendMetaData().withName("backend")); qInstance.addTable(new QTableMetaData().withName("table").withBackendName("backend").withPrimaryKeyField("id").withField(new QFieldMetaData("id", QFieldType.INTEGER))); return qInstance; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 20f87686..58e7ccca 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -27,6 +27,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.enrichment.testplugins.TestEnricherPlugin; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; @@ -47,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_GREETINGS; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_MISCELLANEOUS; @@ -66,6 +69,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class QInstanceEnricherTest extends BaseTest { + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QInstanceEnricher.removeAllEnricherPlugins(); + } + + + /******************************************************************************* ** Test that a table missing a label gets the default label applied (name w/ UC-first). ** @@ -572,4 +586,41 @@ class QInstanceEnricherTest extends BaseTest assertEquals("My Field", qInstance.getProcess("test").getFrontendStep("screen").getViewFields().get(0).getLabel()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldPlugIn() + { + QInstance qInstance = TestUtils.defineInstance(); + + QInstanceEnricher.addEnricherPlugin(new TestEnricherPlugin()); + + new QInstanceEnricher(qInstance).enrich(); + + qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + qInstance.getProcesses().values().forEach(process -> process.getInputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + qInstance.getProcesses().values().forEach(process -> process.getOutputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDiscoverAndAddPlugins() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + new QInstanceEnricher(qInstance).enrich(); + qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).doesNotEndWith("Plugged"))); + + qInstance = TestUtils.defineInstance(); + QInstanceEnricher.discoverAndAddPluginsInPackage(getClass().getPackageName() + ".enrichment.testplugins"); + new QInstanceEnricher(qInstance).enrich(); + qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java index 51af9eaa..3511326f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java @@ -47,6 +47,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataIn import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; @@ -292,6 +294,49 @@ class QInstanceHelpContentManagerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWildcardProcessField() throws QException + { + ///////////////////////////////////// + // get the instance from base test // + ///////////////////////////////////// + QInstance qInstance = QContext.getQInstance(); + new HelpContentMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + + HelpContent recordEntity = new HelpContent() + .withId(1) + .withKey("process:*.bulkInsert;step:upload") + .withContent("v1") + .withRole(HelpContentRole.PROCESS_SCREEN.getId()); + new InsertAction().execute(new InsertInput(HelpContent.TABLE_NAME).withRecordEntity(recordEntity)); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now - post-insert customizer should have automatically added help content to the instance - to all bulkInsert processes // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int hitCount = 0; + for(QTableMetaData table : qInstance.getTables().values()) + { + QProcessMetaData process = qInstance.getProcess(table.getName() + ".bulkInsert"); + if(process == null) + { + return; + } + + List helpContents = process.getFrontendStep("upload").getHelpContents(); + assertEquals(1, helpContents.size()); + assertEquals("v1", helpContents.get(0).getContent()); + assertEquals(Set.of(QHelpRole.PROCESS_SCREEN), helpContents.get(0).getRoles()); + hitCount++; + } + + assertThat(hitCount).isGreaterThanOrEqualTo(3); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -411,7 +456,7 @@ class QInstanceHelpContentManagerTest extends BaseTest QInstanceHelpContentManager.processHelpContentRecord(qInstance, helpContentCreator.apply("foo;bar:baz")); assertThat(collectingLogger.getCollectedMessages()).hasSize(1); - assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("Discarding help content with key that does not contain name:value format"); + assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("Discarding help content with key-part that does not contain name:value format"); collectingLogger.clear(); QInstanceHelpContentManager.processHelpContentRecord(qInstance, helpContentCreator.apply(null)); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 65198111..8419b05a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; @@ -64,6 +66,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValue import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueRangeBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; @@ -92,12 +95,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -181,6 +187,143 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBackendVariants() + { + BackendVariantSetting setting = new BackendVariantSetting() {}; + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true)), + "Missing backendVariantsConfig in backend [variant] which is marked as usesVariants"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(false) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Should not have a backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(null) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Should not have a backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig())), + "Missing variantTypeKey in backendVariantsConfig", + "Missing optionsTableName in backendVariantsConfig", + "Missing or empty backendSettingSourceFieldNameMap"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName("notATable") + .withBackendSettingSourceFieldNameMap(Map.of(setting, "field")))), + "Unrecognized optionsTableName [notATable] in backendVariantsConfig"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withOptionsFilter(new QQueryFilter(new QFilterCriteria("notAField", QCriteriaOperator.EQUALS, 1))) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")))), + "optionsFilter in backendVariantsConfig in backend [variant]: Criteria fieldName notAField is not a field"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "noSuchField")))), + "Unrecognized fieldName [noSuchField] in backendSettingSourceFieldNameMap"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(CustomizerThatIsNotOfTheRightBaseClass.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + )), + "VariantRecordSupplier in backendVariantsConfig in backend [variant]: CodeReference is not any of the expected types: com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction, java.util.function.Function"); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")) + ))); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(VariantRecordFunction.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + ))); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(VariantRecordUnsafeFunction.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + ))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class VariantRecordFunction implements Function + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QRecord apply(Serializable serializable) + { + return null; + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class VariantRecordUnsafeFunction implements UnsafeFunction + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QRecord apply(Serializable serializable) throws QException + { + return null; + } + } + + + /******************************************************************************* ** Test an instance with null tables - should throw. ** @@ -1185,10 +1328,12 @@ public class QInstanceValidatorTest extends BaseTest "should not have searchFields", "should not have orderByFields", "should not have a customCodeReference", - "is missing enum values"); + "is missing enum values", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_STATE).setEnumValues(new ArrayList<>()), - "is missing enum values"); + "is missing enum values", + "is missing its idType."); } @@ -1213,10 +1358,12 @@ public class QInstanceValidatorTest extends BaseTest "should not have a customCodeReference", "is missing a tableName", "is missing searchFields", - "is missing orderByFields"); + "is missing orderByFields", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE).setTableName("Not a table"), - "Unrecognized table"); + "Unrecognized table", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE).setSearchFields(List.of("id", "notAField", "name")), "unrecognized searchField: notAField"); @@ -1244,11 +1391,13 @@ public class QInstanceValidatorTest extends BaseTest "should not have a tableName", "should not have searchFields", "should not have orderByFields", - "is missing a customCodeReference"); + "is missing a customCodeReference", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_CUSTOM).setCustomCodeReference(new QCodeReference()), "missing a code reference name", - "missing a code type"); + "missing a code type", + "is missing its idType."); } @@ -1920,6 +2069,20 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldBehaviorsWithTheirOwnValidateMethods() + { + Function fieldExtractor = qInstance -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("id"); + assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance).withBehavior(new ValueRangeBehavior())), + "Field id in table person: ValueRangeBehavior: Either minValue or maxValue (or both) must be set."); + + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -2348,7 +2511,7 @@ public class QInstanceValidatorTest extends BaseTest { int noOfReasons = actualReasons == null ? 0 : actualReasons.size(); assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons) - + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); + + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); } for(String reason : expectedReasons) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java new file mode 100644 index 00000000..9e940284 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.enrichment.testplugins; + + +import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestEnricherPlugin implements QInstanceEnricherPluginInterface +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void enrich(QFieldMetaData field, QInstance qInstance) + { + if(field != null) + { + field.setLabel(field.getLabel() + " Plugged"); + } + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java new file mode 100644 index 00000000..a0bf6a36 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** AssertJ assert class for ProcessSummary - that is - a list of ProcessSummaryLineInterface's + *******************************************************************************/ +public class ProcessSummaryAssert extends AbstractAssert> +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected ProcessSummaryAssert(List actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static ProcessSummaryAssert assertThat(RunProcessOutput runProcessOutput) + { + List processResults = (List) runProcessOutput.getValue("processResults"); + if(processResults == null) + { + processResults = (List) runProcessOutput.getValue("validationSummary"); + } + + return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static ProcessSummaryAssert assertThat(RunBackendStepOutput runBackendStepOutput) + { + List processResults = (List) runBackendStepOutput.getValue("processResults"); + if(processResults == null) + { + processResults = (List) runBackendStepOutput.getValue("validationSummary"); + } + + if(processResults == null) + { + fail("Could not find process results in backend step output."); + } + + return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryAssert assertThat(List actual) + { + return (new ProcessSummaryAssert(actual, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryAssert hasSize(int expectedSize) + { + Assertions.assertThat(actual).hasSize(expectedSize); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithMessageMatching(String regExp) + { + List foundMessages = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(processSummaryLineInterface.getMessage() == null) + { + processSummaryLineInterface.prepareForFrontend(false); + } + + if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().matches(regExp)) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundMessages.add(processSummaryLineInterface.getMessage()); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with message matching [" + regExp + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithMessageContaining(String substr) + { + List foundMessages = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(processSummaryLineInterface.getMessage() == null) + { + processSummaryLineInterface.prepareForFrontend(false); + } + + if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().contains(substr)) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundMessages.add(processSummaryLineInterface.getMessage()); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with message containing [" + substr + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithStatus(Status status) + { + List foundStatuses = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(status.equals(processSummaryLineInterface.getStatus())) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundStatuses.add(String.valueOf(processSummaryLineInterface.getStatus())); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with status [" + status + "].\nFound statuses were:\n" + StringUtils.join("\n", foundStatuses)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryAssert hasNoLineWithStatus(Status status) + { + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(status.equals(processSummaryLineInterface.getStatus())) + { + failWithMessage("Found a ProcessSummaryLine with status [" + status + "], which was not supposed to happen."); + return (null); + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java new file mode 100644 index 00000000..60d4561c --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** AssertJ assert class for ProcessSummaryLine. + *******************************************************************************/ +public class ProcessSummaryLineInterfaceAssert extends AbstractAssert +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected ProcessSummaryLineInterfaceAssert(ProcessSummaryLineInterface actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryLineInterfaceAssert assertThat(ProcessSummaryLineInterface actual) + { + return (new ProcessSummaryLineInterfaceAssert(actual, ProcessSummaryLineInterfaceAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasCount(Integer count) + { + if(actual instanceof ProcessSummaryLine psl) + { + assertEquals(count, psl.getCount(), "Expected count in process summary line"); + } + else + { + failWithMessage("ProcessSummaryLineInterface is not of concrete type ProcessSummaryLine (is: " + actual.getClass().getSimpleName() + ")"); + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasStatus(Status status) + { + assertEquals(status, actual.getStatus(), "Expected status in process summary line"); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasMessageMatching(String regExp) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).matches(regExp); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasMessageContaining(String substring) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).contains(substring); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveMessageMatching(String regExp) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).doesNotMatch(regExp); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveMessageContaining(String substring) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).doesNotContain(substring); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasAnyBulletsOfTextContaining(String substring) + { + if(actual instanceof ProcessSummaryLine psl) + { + Assertions.assertThat(psl.getBulletsOfText()) + .isNotNull() + .anyMatch(s -> s.contains(substring)); + } + else + { + Assertions.fail("Process Summary Line was not the expected type."); + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveAnyBulletsOfTextContaining(String substring) + { + if(actual instanceof ProcessSummaryLine psl) + { + if(psl.getBulletsOfText() != null) + { + Assertions.assertThat(psl.getBulletsOfText()) + .noneMatch(s -> s.contains(substring)); + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java index 4655255d..affa9325 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java @@ -66,7 +66,7 @@ class NowWithOffsetTest extends BaseTest assertThat(twoWeeksFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), allowedDiff); long oneMonthAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli(); - assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusOneDay); + assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusTwoDays); // two days, to work on 3/1... long twoMonthsFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli(); assertThat(twoMonthsFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), allowedDiffPlusTwoDays); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java index f038c27c..38beaae4 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java @@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.core.model.data; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.testentities.Item; import com.kingsrook.qqq.backend.core.model.data.testentities.ItemWithPrimitives; @@ -35,7 +37,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -49,6 +54,31 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class QRecordEntityTest extends BaseTest { + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QContext.getQInstance().addTable(new QTableMetaData() + .withName(Item.TABLE_NAME) + .withFieldsFromEntity(Item.class) + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QContext.getQInstance().getTables().remove(Item.TABLE_NAME); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -68,6 +98,19 @@ class QRecordEntityTest extends BaseTest assertEquals(47, qRecord.getValueInteger("quantity")); assertEquals(new BigDecimal("3.50"), qRecord.getValueBigDecimal("price")); assertTrue(qRecord.getValueBoolean("featured")); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that, if we had no lists of associations in the entity, that we also have none in the record // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + assertThat(qRecord.getAssociatedRecords()).isNullOrEmpty(); + + /////////////////////////////////////////////////////////////////////// + // now assert that an empty list translates through to an empty list // + /////////////////////////////////////////////////////////////////////// + item.setItemAlternates(Collections.emptyList()); + qRecord = item.toQRecord(); + assertTrue(qRecord.getAssociatedRecords().containsKey(Item.ASSOCIATION_ITEM_ALTERNATES_NAME)); + assertTrue(qRecord.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME).isEmpty()); } @@ -76,9 +119,40 @@ class QRecordEntityTest extends BaseTest ** *******************************************************************************/ @Test - void testItemToQRecordOnlyChangedFields() throws QException + void testItemToQRecordWithAssociations() throws QException + { + Item item = new Item(); + item.setSku("ABC-123"); + item.setQuantity(47); + item.setItemAlternates(List.of( + new Item().withSku("DEF"), + new Item().withSku("GHI").withQuantity(3) + )); + + QRecord qRecord = item.toQRecord(); + assertEquals("ABC-123", qRecord.getValueString("sku")); + assertEquals(47, qRecord.getValueInteger("quantity")); + + List associatedRecords = qRecord.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME); + assertEquals(2, associatedRecords.size()); + assertEquals("DEF", associatedRecords.get(0).getValue("sku")); + assertTrue(associatedRecords.get(0).getValues().containsKey("quantity")); + assertNull(associatedRecords.get(0).getValue("quantity")); + assertEquals("GHI", associatedRecords.get(1).getValue("sku")); + assertEquals(3, associatedRecords.get(1).getValue("quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("deprecation") + @Test + void testItemToQRecordOnlyChangedFieldsEntityThatCameFromQRecord() throws QException { Item item = new Item(new QRecord() + .withValue("id", 1701) .withValue("sku", "ABC-123") .withValue("description", null) .withValue("quantity", 47) @@ -88,11 +162,20 @@ class QRecordEntityTest extends BaseTest QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(); assertTrue(qRecordOnlyChangedFields.getValues().isEmpty()); + QRecord qRecordOnlyChangedFieldsIncludePKey = item.toQRecordOnlyChangedFields(true); + assertEquals(1, qRecordOnlyChangedFieldsIncludePKey.getValues().size()); + assertEquals(1701, qRecordOnlyChangedFieldsIncludePKey.getValue("id")); + item.setDescription("My Changed Item"); - qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(); + qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(false); assertEquals(1, qRecordOnlyChangedFields.getValues().size()); assertEquals("My Changed Item", qRecordOnlyChangedFields.getValueString("description")); + qRecordOnlyChangedFieldsIncludePKey = item.toQRecordOnlyChangedFields(true); + assertEquals(2, qRecordOnlyChangedFieldsIncludePKey.getValues().size()); + assertEquals("My Changed Item", qRecordOnlyChangedFieldsIncludePKey.getValueString("description")); + assertEquals(1701, qRecordOnlyChangedFieldsIncludePKey.getValue("id")); + item.setPrice(null); qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(); assertEquals(2, qRecordOnlyChangedFields.getValues().size()); @@ -101,6 +184,81 @@ class QRecordEntityTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("deprecation") + @Test + void testItemToQRecordOnlyChangedFieldsFromNewEntity() throws QException + { + Item item = new Item() + .withId(1701) + .withSku("ABC-123"); + + QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(); + assertEquals(2, qRecordOnlyChangedFields.getValues().size()); + assertEquals(1701, qRecordOnlyChangedFields.getValue("id")); + assertEquals("ABC-123", qRecordOnlyChangedFields.getValue("sku")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("deprecation") + @Test + void testItemToQRecordOnlyChangedFieldsWithAssociations() throws QException + { + Item item = new Item(new QRecord() + .withValue("id", 1701) + .withValue("sku", "ABC-123") + .withAssociatedRecord(Item.ASSOCIATION_ITEM_ALTERNATES_NAME, new Item(new QRecord() + .withValue("id", 1702) + .withValue("sku", "DEF") + .withValue("quantity", 3) + .withValue("price", new BigDecimal("3.50")) + ).toQRecord()) + ); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if no values were changed in the entities, from when they were constructed (from records), then value maps should be empty // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(false); + assertTrue(qRecordOnlyChangedFields.getValues().isEmpty()); + List associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME); + assertTrue(associatedRecords.get(0).getValues().isEmpty()); + + /////////////////////////////////////////////////////// + // but - if pkeys are requested, confirm we get them // + /////////////////////////////////////////////////////// + qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(true); + assertEquals(1, qRecordOnlyChangedFields.getValues().size()); + assertEquals(1701, qRecordOnlyChangedFields.getValue("id")); + associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME); + assertEquals(1, associatedRecords.get(0).getValues().size()); + assertEquals(1702, associatedRecords.get(0).getValue("id")); + + //////////////////////////////////////////// + // change some properties in the entities // + //////////////////////////////////////////// + item.setDescription("My Changed Item"); + item.getItemAlternates().get(0).setQuantity(4); + item.getItemAlternates().get(0).setPrice(null); + + qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(true); + assertEquals(2, qRecordOnlyChangedFields.getValues().size()); + assertEquals(1701, qRecordOnlyChangedFields.getValue("id")); + assertEquals("My Changed Item", qRecordOnlyChangedFields.getValue("description")); + associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME); + assertEquals(3, associatedRecords.get(0).getValues().size()); + assertEquals(1702, associatedRecords.get(0).getValue("id")); + assertEquals(4, associatedRecords.get(0).getValue("quantity")); + assertNull(associatedRecords.get(0).getValue("price")); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java index 0bd1ddf3..797a93a3 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java @@ -291,4 +291,24 @@ class QRecordTest extends BaseTest assertFalse(jsonObject.has("errorsAsString")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAddJoinedRecordValues() + { + QRecord order = new QRecord().withValue("id", 1).withValue("shipTo", "St. Louis"); + order.addJoinedRecordValues("orderInstructions", null); + assertEquals(2, order.getValues().size()); + + QRecord orderInstructions = new QRecord().withValue("id", 100).withValue("instructions", "Be Careful"); + order.addJoinedRecordValues("orderInstructions", orderInstructions); + + assertEquals(4, order.getValues().size()); + assertEquals(100, order.getValue("orderInstructions.id")); + assertEquals("Be Careful", order.getValue("orderInstructions.instructions")); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithJoinedRecordsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithJoinedRecordsTest.java new file mode 100644 index 00000000..2eb372d0 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithJoinedRecordsTest.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.data; + + +import java.time.LocalDate; +import java.time.Month; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for QRecordWithJoinedRecords + *******************************************************************************/ +class QRecordWithJoinedRecordsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + QRecord order = new QRecord().withValue("id", 1).withValue("orderNo", "101").withValue("orderDate", LocalDate.of(2025, Month.JANUARY, 1)); + QRecord lineItem = new QRecord().withValue("id", 2).withValue("sku", "ABC").withValue("quantity", 47); + QRecord extrinsic = new QRecord().withValue("id", 3).withValue("key", "MyKey").withValue("value", "MyValue"); + + QRecordWithJoinedRecords joinedRecords = new QRecordWithJoinedRecords(order); + joinedRecords.addJoinedRecordValues("lineItem", lineItem); + joinedRecords.addJoinedRecordValues("extrinsic", extrinsic); + + assertEquals(1, joinedRecords.getValue("id")); + assertEquals("101", joinedRecords.getValue("orderNo")); + assertEquals(LocalDate.of(2025, Month.JANUARY, 1), joinedRecords.getValue("orderDate")); + assertEquals(2, joinedRecords.getValue("lineItem.id")); + assertEquals("ABC", joinedRecords.getValue("lineItem.sku")); + assertEquals(47, joinedRecords.getValue("lineItem.quantity")); + assertEquals(3, joinedRecords.getValue("extrinsic.id")); + assertEquals("MyKey", joinedRecords.getValue("extrinsic.key")); + assertEquals("MyValue", joinedRecords.getValue("extrinsic.value")); + + assertEquals(9, joinedRecords.getValues().size()); + assertEquals(1, joinedRecords.getValues().get("id")); + assertEquals(2, joinedRecords.getValues().get("lineItem.id")); + assertEquals(3, joinedRecords.getValues().get("extrinsic.id")); + + joinedRecords.setValue("lineItem.color", "RED"); + assertEquals("RED", joinedRecords.getValue("lineItem.color")); + assertEquals("RED", lineItem.getValue("color")); + + joinedRecords.setValue("shipToCity", "St. Louis"); + assertEquals("St. Louis", joinedRecords.getValue("shipToCity")); + assertEquals("St. Louis", order.getValue("shipToCity")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java index 51be0f3e..7efb8270 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.data.testentities; import java.math.BigDecimal; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QAssociation; import com.kingsrook.qqq.backend.core.model.data.QField; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; @@ -34,6 +36,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; *******************************************************************************/ public class Item extends QRecordEntity { + public static final String TABLE_NAME = "item"; + + public static final String ASSOCIATION_ITEM_ALTERNATES_NAME = "itemAlternates"; + + @QField(isPrimaryKey = true) + private Integer id; + @QField(isRequired = true, label = "SKU") private String sku; @@ -49,6 +58,9 @@ public class Item extends QRecordEntity @QField(backendName = "is_featured") private Boolean featured; + @QAssociation(name = ASSOCIATION_ITEM_ALTERNATES_NAME) + private List itemAlternates; + /******************************************************************************* @@ -179,4 +191,122 @@ public class Item extends QRecordEntity { this.featured = featured; } + + + + /******************************************************************************* + ** Fluent setter for sku + *******************************************************************************/ + public Item withSku(String sku) + { + this.sku = sku; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for description + *******************************************************************************/ + public Item withDescription(String description) + { + this.description = description; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for quantity + *******************************************************************************/ + public Item withQuantity(Integer quantity) + { + this.quantity = quantity; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for price + *******************************************************************************/ + public Item withPrice(BigDecimal price) + { + this.price = price; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for featured + *******************************************************************************/ + public Item withFeatured(Boolean featured) + { + this.featured = featured; + return (this); + } + + + + /******************************************************************************* + ** Getter for itemAlternates + *******************************************************************************/ + public List getItemAlternates() + { + return (this.itemAlternates); + } + + + + /******************************************************************************* + ** Setter for itemAlternates + *******************************************************************************/ + public void setItemAlternates(List itemAlternates) + { + this.itemAlternates = itemAlternates; + } + + + + /******************************************************************************* + ** Fluent setter for itemAlternates + *******************************************************************************/ + public Item withItemAlternates(List itemAlternates) + { + this.itemAlternates = itemAlternates; + return (this); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public Item withId(Integer id) + { + this.id = id; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java new file mode 100644 index 00000000..118c27d3 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for EmptyMetaDataProducerOutput + *******************************************************************************/ +class EmptyMetaDataProducerOutputTest +{ + + /******************************************************************************* + ** sorry, just here to avoid a dip in coverage. + *******************************************************************************/ + @Test + void test() + { + QInstance qInstance = new QInstance(); + new EmptyMetaDataProducerOutput().addSelfToInstance(qInstance); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java index 61ba8d2b..ca574ac3 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelperTest.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.producers.TestMetaDataProdu import com.kingsrook.qqq.backend.core.model.metadata.producers.TestMetaDataProducingPossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.producers.TestNoInterfacesExtendsObject; import com.kingsrook.qqq.backend.core.model.metadata.producers.TestNoValidConstructorMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -76,6 +77,17 @@ class MetaDataProducerHelperTest assertEquals(2, enumPVS.getEnumValues().size()); assertEquals(new QPossibleValue<>(1, "One"), enumPVS.getEnumValues().get(0)); + //////////////////////////////////////////// + // annotation on table -> table meta data // + //////////////////////////////////////////// + assertTrue(qInstance.getTables().containsKey(TestMetaDataProducingEntity.TABLE_NAME)); + QTableMetaData table = qInstance.getTables().get(TestMetaDataProducingEntity.TABLE_NAME); + assertEquals(TestMetaDataProducingEntity.TABLE_NAME, table.getName()); + assertEquals("id", table.getPrimaryKeyField()); + assertEquals(2, table.getFields().size()); + assertTrue(table.getField("name").getIsRequired()); + assertEquals("Customized Label", table.getLabel()); + ////////////////////////////////////////////// // annotation on PVS table -> PVS meta data // ////////////////////////////////////////////// diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueRangeBehaviorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueRangeBehaviorTest.java new file mode 100644 index 00000000..9a1c0362 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueRangeBehaviorTest.java @@ -0,0 +1,144 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.fields; + + +import java.math.BigDecimal; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueRangeBehavior.Behavior; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionAssert; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** Unit test for ValueOutsideOfRangeBehavior + *******************************************************************************/ +class ValueRangeBehaviorTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.getField("noOfShoes").withBehavior(new ValueRangeBehavior().withMinValue(0)); + table.getField("cost").withBehavior(new ValueRangeBehavior().withMaxValue(new BigDecimal("3.50"))); + table.getField("price").withBehavior(new ValueRangeBehavior() + .withMin(BigDecimal.ZERO, false, Behavior.CLIP, new BigDecimal(".01")) + .withMaxValue(new BigDecimal(100)).withMaxAllowEqualTo(false)); + + List recordList = List.of( + new QRecord().withValue("id", 1).withValue("noOfShoes", -1).withValue("cost", new BigDecimal("3.50")).withValue("price", new BigDecimal(-1)), + new QRecord().withValue("id", 2).withValue("noOfShoes", 0).withValue("cost", new BigDecimal("3.51")).withValue("price", new BigDecimal(200)), + new QRecord().withValue("id", 3).withValue("noOfShoes", 1).withValue("cost", new BigDecimal("3.50")).withValue("price", new BigDecimal("99.99")) + ); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList, null); + + { + QRecord record = getRecordById(recordList, 1); + assertEquals(-1, record.getValueInteger("noOfShoes")); // error (but didn't change value) + assertEquals(new BigDecimal("3.50"), record.getValueBigDecimal("cost")); // all okay + assertEquals(new BigDecimal("0.01"), record.getValueBigDecimal("price")); // got clipped + assertThat(record.getErrors()) + .hasSize(1) + .anyMatch(e -> e.getMessage().equals("The value for No Of Shoes is too small (minimum allowed value is 0)")); + } + + { + QRecord record = getRecordById(recordList, 2); + assertEquals(0, record.getValueInteger("noOfShoes")); // all ok + assertEquals(new BigDecimal("3.51"), record.getValueBigDecimal("cost")); // error (but didn't change value) + assertEquals(new BigDecimal(200), record.getValueBigDecimal("price")); // error (but didn't change value) + assertThat(record.getErrors()) + .hasSize(2) + .anyMatch(e -> e.getMessage().equals("The value for Cost is too large (maximum allowed value is 3.50)")) + .anyMatch(e -> e.getMessage().equals("The value for Price is too large (maximum allowed value is less than 100)")); + } + + { + QRecord record = getRecordById(recordList, 3); + assertEquals(1, record.getValueInteger("noOfShoes")); // all ok + assertEquals(new BigDecimal("3.50"), record.getValueBigDecimal("cost")); // all ok + assertEquals(new BigDecimal("99.99"), record.getValueBigDecimal("price")); // all ok + assertThat(record.getErrors()).isNullOrEmpty(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidation() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + QFieldMetaData noOfShoesField = table.getField("noOfShoes"); + QFieldMetaData firstNameField = table.getField("firstName"); + + CollectionAssert.assertThat(new ValueRangeBehavior().validateBehaviorConfiguration(table, noOfShoesField)) + .matchesAll(List.of("Either minValue or maxValue (or both) must be set."), Objects::equals); + + CollectionAssert.assertThat(new ValueRangeBehavior().withMinValue(0).validateBehaviorConfiguration(table, noOfShoesField)).isNullOrEmpty(); + CollectionAssert.assertThat(new ValueRangeBehavior().withMaxValue(100).validateBehaviorConfiguration(table, noOfShoesField)).isNullOrEmpty(); + CollectionAssert.assertThat(new ValueRangeBehavior().withMinValue(0).withMaxValue(100).validateBehaviorConfiguration(table, noOfShoesField)).isNullOrEmpty(); + + CollectionAssert.assertThat(new ValueRangeBehavior().withMinValue(1).withMaxValue(0).validateBehaviorConfiguration(table, noOfShoesField)) + .matchesAll(List.of("minValue must be >= maxValue."), Objects::equals); + + CollectionAssert.assertThat(new ValueRangeBehavior().withMinValue(1).validateBehaviorConfiguration(table, firstNameField)) + .matchesAll(List.of("can only be applied to a numeric type field."), Objects::equals); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QRecord getRecordById(List recordList, Integer id) + { + Optional recordOpt = recordList.stream().filter(r -> r.getValueInteger("id").equals(id)).findFirst(); + if(recordOpt.isEmpty()) + { + fail("Didn't find record with id=" + id); + } + return (recordOpt.get()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestMetaDataProducingEntity.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestMetaDataProducingEntity.java index 519d7946..cb8c903a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestMetaDataProducingEntity.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/producers/TestMetaDataProducingEntity.java @@ -26,7 +26,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QField; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; -import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.ChildJoin; import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.ChildRecordListWidget; @@ -38,7 +37,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; /******************************************************************************* ** QRecord Entity for TestMetaDataProducingEntity table *******************************************************************************/ -@QMetaDataProducingEntity(producePossibleValueSource = true, +@QMetaDataProducingEntity( + produceTableMetaData = true, + tableMetaDataCustomizer = TestMetaDataProducingEntity.TableMetaDataCustomizer.class, + producePossibleValueSource = true, childTables = { @ChildTable(childTableEntityClass = TestMetaDataProducingChildEntity.class, @@ -46,24 +48,31 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; childRecordListWidget = @ChildRecordListWidget(enabled = true, label = "Test Children", maxRows = 15)) } ) -public class TestMetaDataProducingEntity extends QRecordEntity implements MetaDataProducerInterface +public class TestMetaDataProducingEntity extends QRecordEntity { public static final String TABLE_NAME = "testMetaDataProducingEntity"; @QField(isEditable = false, isPrimaryKey = true) private Integer id; + @QField(isRequired = true) + private String name; /*************************************************************************** ** ***************************************************************************/ - @Override - public QTableMetaData produce(QInstance qInstance) throws QException + public static class TableMetaDataCustomizer implements MetaDataCustomizerInterface { - return new QTableMetaData() - .withName(TABLE_NAME) - .withFieldsFromEntity(TestMetaDataProducingEntity.class); + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QTableMetaData customizeMetaData(QInstance qInstance, QTableMetaData table) throws QException + { + table.withLabel("Customized Label"); + return table; + } } @@ -116,4 +125,35 @@ public class TestMetaDataProducingEntity extends QRecordEntity implements MetaDa return (this); } + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public TestMetaDataProducingEntity withName(String name) + { + this.name = name; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java new file mode 100644 index 00000000..de56fbfc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.LinkedHashMap; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata.OtherTableMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata.SomeTableMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for QBitProducer + *******************************************************************************/ +class QBitProducerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + TestQBitConfig config = new TestQBitConfig() + .withOtherTableConfig(ProvidedOrSuppliedTableConfig.provideTableUsingBackendNamed(TestUtils.MEMORY_BACKEND_NAME)) + .withIsSomeTableEnabled(true) + .withSomeSetting("yes") + .withTableMetaDataCustomizer((i, table) -> + { + if(table.getBackendName() == null) + { + table.setBackendName(TestUtils.DEFAULT_BACKEND_NAME); + } + + table.addField(new QFieldMetaData("custom", QFieldType.STRING)); + + return (table); + }); + + QInstance qInstance = QContext.getQInstance(); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // OtherTable should have been provided by the qbit, with the backend name we told it above (MEMORY) // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + QTableMetaData otherTable = qInstance.getTable(OtherTableMetaDataProducer.NAME); + assertNotNull(otherTable); + assertEquals(TestUtils.MEMORY_BACKEND_NAME, otherTable.getBackendName()); + assertNotNull(otherTable.getField("custom")); + + QBitMetaData sourceQBit = otherTable.getSourceQBit(); + assertEquals("testQBit", sourceQBit.getArtifactId()); + + //////////////////////////////////////////////////////////////////////////////// + // SomeTable should have been provided, w/ backend name set by the customizer // + //////////////////////////////////////////////////////////////////////////////// + QTableMetaData someTable = qInstance.getTable(SomeTableMetaDataProducer.NAME); + assertNotNull(someTable); + assertEquals(TestUtils.DEFAULT_BACKEND_NAME, someTable.getBackendName()); + assertNotNull(otherTable.getField("custom")); + + TestQBitConfig qBitConfig = (TestQBitConfig) someTable.getSourceQBitConfig(); + assertEquals("yes", qBitConfig.getSomeSetting()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDisableThings() throws QException + { + TestQBitConfig config = new TestQBitConfig() + .withOtherTableConfig(ProvidedOrSuppliedTableConfig.useSuppliedTaleNamed(TestUtils.TABLE_NAME_PERSON_MEMORY)) + .withIsSomeTableEnabled(false); + + QInstance qInstance = QContext.getQInstance(); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + + ////////////////////////////////////// + // neither table should be produced // + ////////////////////////////////////// + QTableMetaData otherTable = qInstance.getTable(OtherTableMetaDataProducer.NAME); + assertNull(otherTable); + + QTableMetaData someTable = qInstance.getTable(SomeTableMetaDataProducer.NAME); + assertNull(someTable); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidationErrors() throws QException + { + QInstance qInstance = QContext.getQInstance(); + TestQBitConfig config = new TestQBitConfig(); + + assertThatThrownBy(() -> new TestQBitProducer().withTestQBitConfig(config).produce(qInstance)) + .isInstanceOf(QBitConfigValidationException.class) + .hasMessageContaining("otherTableConfig must be set") + .hasMessageContaining("isSomeTableEnabled must be set"); + qInstance.setQBits(new LinkedHashMap<>()); + + config.setIsSomeTableEnabled(true); + assertThatThrownBy(() -> new TestQBitProducer().withTestQBitConfig(config).produce(qInstance)) + .isInstanceOf(QBitConfigValidationException.class) + .hasMessageContaining("otherTableConfig must be set"); + qInstance.setQBits(new LinkedHashMap<>()); + + config.setOtherTableConfig(ProvidedOrSuppliedTableConfig.useSuppliedTaleNamed(TestUtils.TABLE_NAME_PERSON_MEMORY)); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java new file mode 100644 index 00000000..05ac39a9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java @@ -0,0 +1,181 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.ProvidedOrSuppliedTableConfig; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestQBitConfig implements QBitConfig +{ + private MetaDataCustomizerInterface tableMetaDataCustomizer; + + private Boolean isSomeTableEnabled; + private ProvidedOrSuppliedTableConfig otherTableConfig; + + private String someSetting; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void validate(QInstance qInstance, List errors) + { + assertCondition(otherTableConfig != null, "otherTableConfig must be set", errors); + assertCondition(isSomeTableEnabled != null, "isSomeTableEnabled must be set", errors); + } + + + + /******************************************************************************* + ** Getter for otherTableConfig + *******************************************************************************/ + public ProvidedOrSuppliedTableConfig getOtherTableConfig() + { + return (this.otherTableConfig); + } + + + + /******************************************************************************* + ** Setter for otherTableConfig + *******************************************************************************/ + public void setOtherTableConfig(ProvidedOrSuppliedTableConfig otherTableConfig) + { + this.otherTableConfig = otherTableConfig; + } + + + + /******************************************************************************* + ** Fluent setter for otherTableConfig + *******************************************************************************/ + public TestQBitConfig withOtherTableConfig(ProvidedOrSuppliedTableConfig otherTableConfig) + { + this.otherTableConfig = otherTableConfig; + return (this); + } + + + + /******************************************************************************* + ** Getter for isSomeTableEnabled + *******************************************************************************/ + public Boolean getIsSomeTableEnabled() + { + return (this.isSomeTableEnabled); + } + + + + /******************************************************************************* + ** Setter for isSomeTableEnabled + *******************************************************************************/ + public void setIsSomeTableEnabled(Boolean isSomeTableEnabled) + { + this.isSomeTableEnabled = isSomeTableEnabled; + } + + + + /******************************************************************************* + ** Fluent setter for isSomeTableEnabled + *******************************************************************************/ + public TestQBitConfig withIsSomeTableEnabled(Boolean isSomeTableEnabled) + { + this.isSomeTableEnabled = isSomeTableEnabled; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableMetaDataCustomizer + *******************************************************************************/ + public MetaDataCustomizerInterface getTableMetaDataCustomizer() + { + return (this.tableMetaDataCustomizer); + } + + + + /******************************************************************************* + ** Setter for tableMetaDataCustomizer + *******************************************************************************/ + public void setTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + this.tableMetaDataCustomizer = tableMetaDataCustomizer; + } + + + + /******************************************************************************* + ** Fluent setter for tableMetaDataCustomizer + *******************************************************************************/ + public TestQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + this.tableMetaDataCustomizer = tableMetaDataCustomizer; + return (this); + } + + + + /******************************************************************************* + ** Getter for someSetting + *******************************************************************************/ + public String getSomeSetting() + { + return (this.someSetting); + } + + + + /******************************************************************************* + ** Setter for someSetting + *******************************************************************************/ + public void setSomeSetting(String someSetting) + { + this.someSetting = someSetting; + } + + + + /******************************************************************************* + ** Fluent setter for someSetting + *******************************************************************************/ + public TestQBitConfig withSomeSetting(String someSetting) + { + this.someSetting = someSetting; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java new file mode 100644 index 00000000..14c1d5d1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestQBitProducer implements QBitProducer +{ + private TestQBitConfig testQBitConfig; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void produce(QInstance qInstance, String namespace) throws QException + { + QBitMetaData qBitMetaData = new QBitMetaData() + .withGroupId("test.com.kingsrook.qbits") + .withArtifactId("testQBit") + .withVersion("0.1.0") + .withNamespace(namespace) + .withConfig(testQBitConfig); + qInstance.addQBit(qBitMetaData); + + List> producers = MetaDataProducerHelper.findProducers(getClass().getPackageName() + ".metadata"); + finishProducing(qInstance, qBitMetaData, testQBitConfig, producers); + } + + + + /******************************************************************************* + ** Getter for testQBitConfig + *******************************************************************************/ + public TestQBitConfig getTestQBitConfig() + { + return (this.testQBitConfig); + } + + + + /******************************************************************************* + ** Setter for testQBitConfig + *******************************************************************************/ + public void setTestQBitConfig(TestQBitConfig testQBitConfig) + { + this.testQBitConfig = testQBitConfig; + } + + + + /******************************************************************************* + ** Fluent setter for testQBitConfig + *******************************************************************************/ + public TestQBitProducer withTestQBitConfig(TestQBitConfig testQBitConfig) + { + this.testQBitConfig = testQBitConfig; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java new file mode 100644 index 00000000..7660758c --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Meta Data Producer for OtherTable + *******************************************************************************/ +public class OtherTableMetaDataProducer extends QBitComponentMetaDataProducer +{ + public static final String NAME = "otherTable"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (getQBitConfig().getOtherTableConfig().getDoProvideTable()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(NAME) + .withPrimaryKeyField("id") + .withBackendName(getQBitConfig().getOtherTableConfig().getBackendName()) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + + return (qTableMetaData); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java new file mode 100644 index 00000000..3ef79554 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Meta Data Producer for SomeTable + *******************************************************************************/ +public class SomeTableMetaDataProducer extends QBitComponentMetaDataProducer +{ + public static final String NAME = "someTable"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (getQBitConfig().getIsSomeTableEnabled()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + + return (qTableMetaData); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java new file mode 100644 index 00000000..5f406acb --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for SectionFactory + *******************************************************************************/ +class SectionFactoryTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + QFieldSection t1section = SectionFactory.defaultT1("id", "name"); + assertEquals(SectionFactory.getDefaultT1name(), t1section.getName()); + assertEquals(SectionFactory.getDefaultT1iconName(), t1section.getIcon().getName()); + assertEquals(Tier.T1, t1section.getTier()); + assertEquals(List.of("id", "name"), t1section.getFieldNames()); + + QFieldSection t2section = SectionFactory.defaultT2("size", "age"); + assertEquals(SectionFactory.getDefaultT2name(), t2section.getName()); + assertEquals(SectionFactory.getDefaultT2iconName(), t2section.getIcon().getName()); + assertEquals(Tier.T2, t2section.getTier()); + assertEquals(List.of("size", "age"), t2section.getFieldNames()); + + QFieldSection t3section = SectionFactory.defaultT3("createDate", "modifyDate"); + assertEquals(SectionFactory.getDefaultT3name(), t3section.getName()); + assertEquals(SectionFactory.getDefaultT3iconName(), t3section.getIcon().getName()); + assertEquals(Tier.T3, t3section.getTier()); + assertEquals(List.of("createDate", "modifyDate"), t3section.getFieldNames()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java new file mode 100644 index 00000000..4bdbbc06 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for TablesCustomPossibleValueProvider + *******************************************************************************/ +class TablesCustomPossibleValueProviderTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() + { + QInstance qInstance = TestUtils.defineInstance(); + + qInstance.addTable(new QTableMetaData() + .withName("hidden") + .withIsHidden(true) + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER))); + + qInstance.addTable(new QTableMetaData() + .withName("restricted") + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)) + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER))); + + qInstance.addPossibleValueSource(TablesPossibleValueSourceMetaDataProvider.defineTablesPossibleValueSource(qInstance)); + + QContext.init(qInstance, newSession()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetPossibleValue() + { + TablesCustomPossibleValueProvider provider = new TablesCustomPossibleValueProvider(); + + QPossibleValue possibleValue = provider.getPossibleValue(TestUtils.TABLE_NAME_PERSON); + assertEquals(TestUtils.TABLE_NAME_PERSON, possibleValue.getId()); + assertEquals("Person", possibleValue.getLabel()); + + assertNull(provider.getPossibleValue("no-such-table")); + assertNull(provider.getPossibleValue("hidden")); + assertNull(provider.getPossibleValue("restricted")); + + QContext.getQSession().withPermission("restricted.hasAccess"); + assertNotNull(provider.getPossibleValue("restricted")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPossibleValue() throws QException + { + TablesCustomPossibleValueProvider provider = new TablesCustomPossibleValueProvider(); + + List> list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).noneMatch(p -> p.getId().equals("no-such-table")); + assertThat(list).noneMatch(p -> p.getId().equals("hidden")); + assertThat(list).noneMatch(p -> p.getId().equals("restricted")); + assertNull(provider.getPossibleValue("restricted")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withIdList(List.of(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_SHAPE, "hidden"))); + assertEquals(2, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).noneMatch(p -> p.getId().equals("hidden")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withLabelList(List.of("Person", "Shape", "Restricted"))); + assertEquals(2, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).noneMatch(p -> p.getId().equals("restricted")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withSearchTerm("restricted")); + assertEquals(0, list.size()); + + ///////////////////////////////////////// + // add permission for restricted table // + ///////////////////////////////////////// + QContext.getQSession().withPermission("restricted.hasAccess"); + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withSearchTerm("restricted")); + assertEquals(1, list.size()); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withLabelList(List.of("Person", "Shape", "Restricted"))); + assertEquals(3, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).anyMatch(p -> p.getId().equals("restricted")); + + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java new file mode 100644 index 00000000..5957ee61 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java @@ -0,0 +1,100 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for BackendVariantsUtil + *******************************************************************************/ +class BackendVariantsUtilTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetVariantId() throws QException + { + QBackendMetaData myBackend = getBackendMetaData(); + + assertThatThrownBy(() -> BackendVariantsUtil.getVariantId(myBackend)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701)); + assertEquals(1701, BackendVariantsUtil.getVariantId(myBackend)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData getBackendMetaData() + { + QBackendMetaData myBackend = new QBackendMetaData() + .withName("TestBackend") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TestUtils.TABLE_NAME_SHAPE) + .withVariantTypeKey("yourSelectedShape")); + return myBackend; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetVariantRecord() throws QException + { + QBackendMetaData myBackend = getBackendMetaData(); + + TestUtils.insertDefaultShapes(QContext.getQInstance()); + + assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701)); + assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend)) + .hasMessageContaining("Could not find Backend Variant in table shape with id '1701'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1)); + QRecord variantRecord = BackendVariantsUtil.getVariantRecord(myBackend); + assertEquals(1, variantRecord.getValueInteger("id")); + assertNotNull(variantRecord.getValue("name")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessTableManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessTableManagerTest.java new file mode 100644 index 00000000..cf00fa81 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/processes/QQQProcessTableManagerTest.java @@ -0,0 +1,95 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.processes; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for QQQProcessTableManager + *******************************************************************************/ +class QQQProcessTableManagerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testProcessesGetInsertedUponRequest() throws QException + { + new QQQProcessesMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + + Integer greetPeopleProcessId = QQQProcessTableManager.getQQQProcessId(QContext.getQInstance(), TestUtils.PROCESS_NAME_GREET_PEOPLE); + assertEquals(1, greetPeopleProcessId); + + assertEquals(1, QueryAction.execute(QQQProcessesMetaDataProvider.QQQ_PROCESS_CACHE_TABLE_NAME, new QQueryFilter()).size()); + assertEquals(1, QueryAction.execute(QQQProcess.TABLE_NAME, new QQueryFilter()).size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExistingProcessComesBack() throws QException + { + new QQQProcessesMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + + new InsertAction().execute(new InsertInput(QQQProcess.TABLE_NAME).withRecordEntity(new QQQProcess().withName(TestUtils.PROCESS_NAME_GREET_PEOPLE))); + new InsertAction().execute(new InsertInput(QQQProcess.TABLE_NAME).withRecordEntity(new QQQProcess().withName(TestUtils.PROCESS_NAME_ADD_TO_PEOPLES_AGE))); + + assertEquals(0, QueryAction.execute(QQQProcessesMetaDataProvider.QQQ_PROCESS_CACHE_TABLE_NAME, new QQueryFilter()).size()); + assertEquals(2, QueryAction.execute(QQQProcess.TABLE_NAME, new QQueryFilter()).size()); + + assertEquals(2, QQQProcessTableManager.getQQQProcessId(QContext.getQInstance(), TestUtils.PROCESS_NAME_ADD_TO_PEOPLES_AGE)); + + assertEquals(1, QueryAction.execute(QQQProcessesMetaDataProvider.QQQ_PROCESS_CACHE_TABLE_NAME, new QQueryFilter()).size()); + assertEquals(2, QueryAction.execute(QQQProcess.TABLE_NAME, new QQueryFilter()).size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBogusProcessName() throws QException + { + new QQQProcessesMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + assertNull(QQQProcessTableManager.getQQQProcessId(QContext.getQInstance(), "not a process")); + assertEquals(0, QueryAction.execute(QQQProcessesMetaDataProvider.QQQ_PROCESS_CACHE_TABLE_NAME, new QQueryFilter()).size()); + assertEquals(0, QueryAction.execute(QQQProcess.TABLE_NAME, new QQueryFilter()).size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java index cd676342..dd085268 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java @@ -76,7 +76,7 @@ class QSessionTest extends BaseTest void testMixedValueTypes() { QSession session = new QSession().withSecurityKeyValues(Map.of( - "storeId", List.of("100", "200", 300) + "storeId", List.of("100", "200", 300, "four-hundred") )); for(int i : List.of(100, 200, 300)) @@ -86,6 +86,18 @@ class QSessionTest extends BaseTest assertTrue(session.hasSecurityKeyValue("storeId", i, QFieldType.STRING), "Should contain: " + i); assertTrue(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.STRING), "Should contain: " + i); } + + //////////////////////////////////////////////////////////////////////////// + // next two blocks - used to throw exceptions - now, gracefully be false. // + //////////////////////////////////////////////////////////////////////////// + int i = 400; + assertFalse(session.hasSecurityKeyValue("storeId", i, QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", i, QFieldType.STRING), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.STRING), "Should not contain: " + i); + + assertFalse(session.hasSecurityKeyValue("storeId", "one-hundred", QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", "one-hundred", QFieldType.STRING), "Should not contain: " + i); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/tables/QQQTableTableManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/tables/QQQTableTableManagerTest.java new file mode 100644 index 00000000..7d785363 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/tables/QQQTableTableManagerTest.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.tables; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for QQQTableTableManager + *******************************************************************************/ +class QQQTableTableManagerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTablesGetInsertedUponRequest() throws QException + { + new QQQTablesMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + + Integer personMemoryTableId = QQQTableTableManager.getQQQTableId(QContext.getQInstance(), TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals(1, personMemoryTableId); + + assertEquals(1, QueryAction.execute(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME, new QQueryFilter()).size()); + assertEquals(1, QueryAction.execute(QQQTable.TABLE_NAME, new QQueryFilter()).size()); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExistingTableComesBack() throws QException + { + new QQQTablesMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + + new InsertAction().execute(new InsertInput(QQQTable.TABLE_NAME).withRecordEntity(new QQQTable().withName(TestUtils.TABLE_NAME_SHAPE))); + new InsertAction().execute(new InsertInput(QQQTable.TABLE_NAME).withRecordEntity(new QQQTable().withName(TestUtils.TABLE_NAME_ID_AND_NAME_ONLY))); + + assertEquals(0, QueryAction.execute(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME, new QQueryFilter()).size()); + assertEquals(2, QueryAction.execute(QQQTable.TABLE_NAME, new QQueryFilter()).size()); + + assertEquals(2, QQQTableTableManager.getQQQTableId(QContext.getQInstance(), TestUtils.TABLE_NAME_ID_AND_NAME_ONLY)); + + assertEquals(1, QueryAction.execute(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME, new QQueryFilter()).size()); + assertEquals(2, QueryAction.execute(QQQTable.TABLE_NAME, new QQueryFilter()).size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBogusTableName() throws QException + { + new QQQTablesMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + assertNull(QQQTableTableManager.getQQQTableId(QContext.getQInstance(), "not a table")); + assertEquals(0, QueryAction.execute(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME, new QQueryFilter()).size()); + assertEquals(0, QueryAction.execute(QQQTable.TABLE_NAME, new QQueryFilter()).size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java index afc3ad1f..1f922a2e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java @@ -22,8 +22,10 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.utils; +import java.io.Serializable; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaOption; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -305,6 +307,73 @@ class BackendQueryFilterUtilsTest + /*************************************************************************** + ** + ***************************************************************************/ + private QFilterCriteria newCaseInsensitiveCriteria(String fieldName, QCriteriaOperator operator, Serializable... values) + { + return new QFilterCriteria(fieldName, operator, values).withOption(CriteriaOption.CASE_INSENSITIVE); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QFilterCriteria newCaseInsensitiveCriteria(String fieldName, QCriteriaOperator operator, List values) + { + return new QFilterCriteria(fieldName, operator, values).withOption(CriteriaOption.CASE_INSENSITIVE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDoesCriterionMatchCaseInsensitive() + { + //////////////// + // like & not // + //////////////// + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.LIKE, "Test"), "f", "test")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.LIKE, "test"), "f", "Test")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.LIKE, "T%"), "f", "test")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.LIKE, "t%"), "f", "Test")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.LIKE, "T_st"), "f", "test")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.LIKE, "t_st"), "f", "Test")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_LIKE, "Test"), "f", "Tst")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_LIKE, "Test"), "f", "tst")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_LIKE, "T%"), "f", "Rest")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_LIKE, "T_st"), "f", "Toast")); + + ////////////// + // IN & NOT // + ////////////// + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.IN, "A"), "f", "a")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.IN, "a"), "f", "A")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.IN, "A", "B"), "f", "a")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.IN, "A", "b"), "f", "B")); + assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.IN, List.of()), "f", "A")); + assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.IN, ListBuilder.of(null)), "f", "A")); + + assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_IN, "A"), "f", "A")); + assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_IN, "A", "B"), "f", "a")); + assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_IN, "A", "b"), "f", "B")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_IN, List.of()), "f", "A")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_IN, ListBuilder.of(null)), "f", "A")); + + /////////////////////////// + // NOT_EQUALS_OR_IS_NULL // + /////////////////////////// + assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, "A"), "f", "A")); + assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, "A"), "f", "a")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, "A"), "f", "B")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(newCaseInsensitiveCriteria("f", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, "A"), "f", null)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java new file mode 100644 index 00000000..2fe422fd --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -0,0 +1,334 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.OutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for full bulk insert process + *******************************************************************************/ +class BulkInsertFullProcessTest extends BaseTest +{ + private static final String defaultEmail = "noone@kingsrook.com"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow1() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow2() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com","Illinois", + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvHeaderUsingLabels() + { + return (""" + "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email","Home State",noOfShoes + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws Exception + { + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + ///////////////////////////////////////////////////////// + // start the process - expect to go to the upload step // + ///////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); + + ////////////////////////// + // continue post-upload // + ////////////////////////// + runProcessOutput = continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(2)); + assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); + assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); + + ////////////////////////////////////////////////////// + // assert about the suggested mapping that was done // + ////////////////////////////////////////////////////// + Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile"); + assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class); + assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5); + assertEquals("firstName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); + assertEquals(3, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertEquals("lastName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getFieldName()); + assertEquals(4, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getColumnIndex()); + assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getFieldName()); + assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getColumnIndex()); + + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); + + //////////////////////////////// + // continue post file-mapping // + //////////////////////////////// + runProcessOutput = continueProcessPostFileMapping(runProcessInput); + Serializable valueMappingField = runProcessOutput.getValue("valueMappingField"); + assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class); + assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); + assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues")); + assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping")); + assertEquals(Map.of(1, "IL"), runProcessOutput.getValue("mappedValueLabels")); + assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex")); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping"); + + ///////////////////////////////// + // continue post value-mapping // + ///////////////////////////////// + runProcessOutput = continueProcessPostValueMapping(runProcessInput); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + ///////////////////////////////// + // continue post review screen // + ///////////////////////////////// + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2"); + + //////////////////////////////////// + // query for the inserted records // + //////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + assertEquals("John", records.get(0).getValueString("firstName")); + assertEquals("Jane", records.get(1).getValueString("firstName")); + + assertNotNull(records.get(0).getValue("id")); + assertNotNull(records.get(1).getValue("id")); + + assertEquals(2, records.get(0).getValueInteger("homeStateId")); + assertEquals(1, records.get(1).getValueInteger("homeStateId")); + + assertEquals(defaultEmail, records.get(0).getValueString("email")); + assertEquals(defaultEmail, records.get(1).getValueString("email")); + + assertEquals(42, records.get(0).getValueInteger("noOfShoes")); + assertNull(records.get(1).getValue("noOfShoes")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneRow() throws Exception + { + /////////////////////////////////////// + // make sure table is empty to start // + /////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(1)); + continueProcessPostFileMapping(runProcessInput); + continueProcessPostValueMapping(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // all that just so we can make sure this message is right (because it was wrong when we first wrote it, lol) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id 1"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostReviewScreen(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("review"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostValueMapping(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setStartAfterStep("valueMapping"); + runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); + addProfileToRunProcessInput(runProcessInput); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostFileMapping(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("fileMapping"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostUpload(RunProcessInput runProcessInput, String processUUID, StorageInput storageInput) throws QException + { + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static StorageInput simulateFileUpload(int noOfRows) throws Exception + { + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + (noOfRows == 2 ? getPersonCsvRow2() : "")).getBytes()); + } + return storageInput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput startProcess(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addProfileToRunProcessInput(RunProcessInput input) + { + input.addValue("version", "v1"); + input.addValue("layout", "FLAT"); + input.addValue("hasHeaderRow", "true"); + input.addValue("fieldListJSON", JsonUtils.toJson(List.of( + new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), + new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), + new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), + new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) + ))); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java new file mode 100644 index 00000000..ea6e0e8f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for BulkInsertPrepareMappingStep + *******************************************************************************/ +class BulkInsertPrepareFileMappingStepTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("PointlessArithmeticExpression") + @Test + void testToHeaderLetter() + { + assertEquals("A", BulkInsertPrepareFileMappingStep.toHeaderLetter(0)); + assertEquals("B", BulkInsertPrepareFileMappingStep.toHeaderLetter(1)); + assertEquals("Z", BulkInsertPrepareFileMappingStep.toHeaderLetter(25)); + + assertEquals("AA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 0)); + assertEquals("AB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 1)); + assertEquals("AZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 25)); + + assertEquals("BA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 0)); + assertEquals("BB", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 1)); + assertEquals("BZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 25)); + + assertEquals("ZA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 0)); + assertEquals("ZB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 1)); + assertEquals("ZZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 25)); + + assertEquals("AAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 0)); + assertEquals("AAB", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 1)); + assertEquals("AAC", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 2)); + + assertEquals("ABA", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 0)); + assertEquals("ABB", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 1)); + + assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java new file mode 100644 index 00000000..6953c5bc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for BulkInsertPrepareValueMappingStep + *******************************************************************************/ +class BulkInsertPrepareValueMappingStepTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + assertEquals(TestUtils.TABLE_NAME_ORDER, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").table().getName()); + assertEquals("orderNo", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").field().getName()); + + assertEquals(TestUtils.TABLE_NAME_LINE_ITEM, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").table().getName()); + assertEquals("sku", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").field().getName()); + + assertEquals(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").table().getName()); + assertEquals("key", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").field().getName()); + + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java deleted file mode 100644 index 496ad0be..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - - -/******************************************************************************* - ** Unit test for full bulk insert process - *******************************************************************************/ -class BulkInsertTest extends BaseTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @BeforeEach - @AfterEach - void beforeAndAfterEach() - { - MemoryRecordStore.getInstance().reset(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow1() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow2() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvHeaderUsingLabels() - { - return (""" - "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - /////////////////////////////////////// - // make sure table is empty to start // - /////////////////////////////////////// - assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); - - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); - qUploadedFile.setFilename("test.csv"); - UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile); - - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); - RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); - String processUUID = runProcessOutput.getProcessUUID(); - - runProcessInput.setProcessUUID(processUUID); - runProcessInput.setStartAfterStep("upload"); - runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, uploadedFileKey); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); - - runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); - runProcessInput.setStartAfterStep("review"); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); - assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); - - runProcessInput.setStartAfterStep("review"); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); - assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); - assertThat(runProcessOutput.getException()).isEmpty(); - - //////////////////////////////////// - // query for the inserted records // - //////////////////////////////////// - List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); - assertEquals("John", records.get(0).getValueString("firstName")); - assertEquals("Jane", records.get(1).getValueString("firstName")); - assertNotNull(records.get(0).getValue("id")); - assertNotNull(records.get(1).getValue("id")); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java index b87b6799..386050eb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java @@ -22,10 +22,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -37,9 +41,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -87,9 +96,9 @@ class BulkInsertTransformStepTest extends BaseTest // insert some records that will cause some UK violations // //////////////////////////////////////////////////////////// TestUtils.insertRecords(table, List.of( - newQRecord("uuid-A", "SKU-1", 1), - newQRecord("uuid-B", "SKU-2", 1), - newQRecord("uuid-C", "SKU-2", 2) + newUkTestQRecord("uuid-A", "SKU-1", 1), + newUkTestQRecord("uuid-B", "SKU-2", 1), + newUkTestQRecord("uuid-C", "SKU-2", 2) )); /////////////////////////////////////////// @@ -102,13 +111,13 @@ class BulkInsertTransformStepTest extends BaseTest input.setTableName(TABLE_NAME); input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); input.setRecords(List.of( - newQRecord("uuid-1", "SKU-A", 1), // OK. - newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set - newQRecord("uuid-2", "SKU-C", 1), // OK. - newQRecord("uuid-3", "SKU-C", 2), // OK. - newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set - newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records - newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + newUkTestQRecord("uuid-1", "SKU-A", 1), // OK. + newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newUkTestQRecord("uuid-2", "SKU-C", 1), // OK. + newUkTestQRecord("uuid-3", "SKU-C", 2), // OK. + newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); bulkInsertTransformStep.runOnePage(input, output); @@ -171,9 +180,9 @@ class BulkInsertTransformStepTest extends BaseTest // insert some records that will cause some UK violations // //////////////////////////////////////////////////////////// TestUtils.insertRecords(table, List.of( - newQRecord("uuid-A", "SKU-1", 1), - newQRecord("uuid-B", "SKU-2", 1), - newQRecord("uuid-C", "SKU-2", 2) + newUkTestQRecord("uuid-A", "SKU-1", 1), + newUkTestQRecord("uuid-B", "SKU-2", 1), + newUkTestQRecord("uuid-C", "SKU-2", 2) )); /////////////////////////////////////////// @@ -186,20 +195,20 @@ class BulkInsertTransformStepTest extends BaseTest input.setTableName(TABLE_NAME); input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); input.setRecords(List.of( - newQRecord("uuid-1", "SKU-A", 1), // OK. - newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set - newQRecord("uuid-2", "SKU-C", 1), // OK. - newQRecord("uuid-3", "SKU-C", 2), // OK. - newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set - newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records - newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + newUkTestQRecord("uuid-1", "SKU-A", 1), // OK. + newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newUkTestQRecord("uuid-2", "SKU-C", 1), // OK. + newUkTestQRecord("uuid-3", "SKU-C", 2), // OK. + newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); bulkInsertTransformStep.runOnePage(input, output); - /////////////////////////////////////////////////////// - // assert that all records pass. - /////////////////////////////////////////////////////// + /////////////////////////////////// + // assert that all records pass. // + /////////////////////////////////// assertEquals(7, output.getRecords().size()); } @@ -211,8 +220,8 @@ class BulkInsertTransformStepTest extends BaseTest private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId) { return (record.getValue("uuid").equals(uuid) - && record.getValue("sku").equals(sku) - && record.getValue("storeId").equals(storeId)); + && record.getValue("sku").equals(sku) + && record.getValue("storeId").equals(storeId)); } @@ -220,7 +229,7 @@ class BulkInsertTransformStepTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private QRecord newQRecord(String uuid, String sku, int storeId) + private QRecord newUkTestQRecord(String uuid, String sku, int storeId) { return new QRecord() .withValue("uuid", uuid) @@ -229,4 +238,191 @@ class BulkInsertTransformStepTest extends BaseTest .withValue("name", "Some Item"); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValueMappingTypeErrors() throws QException + { + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + Serializable[] emptyValues = new Serializable[0]; + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 1)) + .withError(new BulkLoadValueTypeError("storeId", "A", QFieldType.INTEGER, "Store")) + .withError(new BulkLoadValueTypeError("orderDate", "47", QFieldType.DATE, "Order Date")), + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 2)) + .withError(new BulkLoadValueTypeError("storeId", "BCD", QFieldType.INTEGER, "Store")) + )); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // add 102 records with an error in the total field - which is more than the number of examples that should be given // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(int i = 0; i < 102; i++) + { + input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i)) + .withError(new BulkLoadValueTypeError("total", "three-fifty-" + i, QFieldType.DECIMAL, "Total"))); + } + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Store] to type [Integer]") + .hasMessageContaining("Values:") + .doesNotHaveMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 1 [A]") + .hasAnyBulletsOfTextContaining("Row 2 [BCD]") + .hasStatus(Status.ERROR) + .hasCount(2); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Order Date] to type [Date]") + .hasMessageContaining("Values:") + .doesNotHaveMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 1 [47]") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Total] to type [Decimal]") + .hasMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 3 [three-fifty-0]") + .hasAnyBulletsOfTextContaining("Row 4 [three-fifty-1]") + .hasAnyBulletsOfTextContaining("Row 5 [three-fifty-2]") + .doesNotHaveAnyBulletsOfTextContaining("three-fifty-101") + .hasStatus(Status.ERROR) + .hasCount(102); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRollupOfValidationErrors() throws QException + { + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1); + + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + Serializable[] emptyValues = new Serializable[0]; + + String tooLong = ".".repeat(201); + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", tooLong), new BulkLoadFileRow(emptyValues, 1)), + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", "OK").withValue("storeId", 1), new BulkLoadFileRow(emptyValues, 2)) + )); + + ///////////////////////////////////////////////////////////////////// + // add 102 records with no security key - which should be an error // + ///////////////////////////////////////////////////////////////////// + for(int i = 0; i < 102; i++) + { + input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i))); + } + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("value for Ship To Name is too long") + .hasMessageContaining("Records:") + .doesNotHaveMessageContaining("Example Records:") + .hasAnyBulletsOfTextContaining("Row 1") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("without a value in the field: Store Id") + .hasMessageContaining("Example Records:") + .hasAnyBulletsOfTextContaining("Row 1") + .hasAnyBulletsOfTextContaining("Row 3") + .hasAnyBulletsOfTextContaining("Row 4") + .doesNotHaveAnyBulletsOfTextContaining("Row 101") + .hasStatus(Status.ERROR) + .hasCount(103); // the 102, plus row 1. + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Order record will be inserted") + .hasStatus(Status.OK) + .hasCount(1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPropagationOfErrorsFromAssociations() throws QException + { + //////////////////////////////////////////////// + // set line item lineNumber field as required // + //////////////////////////////////////////////// + QInstance instance = TestUtils.defineInstance(); + instance.getTable(TestUtils.TABLE_NAME_LINE_ITEM).getField("lineNumber").setIsRequired(true); + reInitInstanceInContext(instance); + + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1); + + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + new QRecord().withValue("storeId", 1).withAssociatedRecord("orderLine", new QRecord()), + new QRecord().withValue("storeId", 1).withAssociatedRecord("orderLine", new QRecord().withError(new BadInputStatusMessage("some mapping error"))) + )); + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("some mapping error") + .hasMessageContaining("Records:") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("records were processed from the file") + .hasStatus(Status.INFO) + .hasCount(2); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Order record will be inserted") + .hasStatus(Status.OK) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Order Line record will be inserted") + .hasStatus(Status.OK) + .hasCount(1); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java new file mode 100644 index 00000000..07a82fed --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** Unit test for CsvFileToRows + *******************************************************************************/ +class CsvFileToRowsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + byte[] csvBytes = """ + one,two,three + 1,2,3,4 + """.getBytes(); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.csv", new ByteArrayInputStream(csvBytes)); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + + assertEquals(new BulkLoadFileRow(new String[] { "one", "two", "three" }, 1), headerRow); + assertEquals(new BulkLoadFileRow(new String[] { "1", "2", "3", "4" }, 2), bodyRow); + assertFalse(fileToRowsInterface.hasNext()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java new file mode 100644 index 00000000..a5642de2 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; + + +/*************************************************************************** + ** + ***************************************************************************/ +public class TestFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private final List rows; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TestFileToRows(List rows) + { + this.rows = rows; + setIterator(this.rows.iterator()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + /////////// + // noop! // + /////////// + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + /////////// + // noop! // + /////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(Serializable[] values) + { + return (new BulkLoadFileRow(values, getRowNo())); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java new file mode 100644 index 00000000..8d144ad6 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -0,0 +1,161 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.time.LocalDate; +import java.time.Month; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for XlsxFileToRows + *******************************************************************************/ +class XlsxFileToRowsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException, IOException + { + byte[] byteArray = writeExcelBytes(); + + FileToRowsInterface fileToRowsInterface = new XlsxFileToRows(); + fileToRowsInterface.init(new ByteArrayInputStream(byteArray)); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + + assertEquals(new BulkLoadFileRow(new String[] { "Id", "First Name", "Last Name", "Birth Date" }, 1), headerRow); + assertEquals(new BulkLoadFileRow(new Serializable[] { 1, "Darin", "Jonson", LocalDate.of(1980, Month.JANUARY, 31) }, 2), bodyRow); + + /////////////////////////////////////////////////////////////////////////////////////// + // make sure there's at least a limit (less than 20) to how many more rows there are // + /////////////////////////////////////////////////////////////////////////////////////// + int otherRowCount = 0; + while(fileToRowsInterface.hasNext() && otherRowCount < 20) + { + fileToRowsInterface.next(); + otherRowCount++; + } + assertFalse(fileToRowsInterface.hasNext()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static byte[] writeExcelBytes() throws QException, IOException + { + ReportFormat format = ReportFormat.XLSX; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(GenerateReportActionTest.defineTableOnlyReport()); + GenerateReportActionTest.insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(baos)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + reportInput.setOverrideExportStreamerSupplier(ExcelFastexcelExportStreamer::new); + new GenerateReportAction().execute(reportInput); + + byte[] byteArray = baos.toByteArray(); + // FileUtils.writeByteArrayToFile(new File("/tmp/xlsx.xlsx"), byteArray); + return byteArray; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDateTimeFormats() + { + assertFormatDateAndOrDateTime(true, false, "dddd, m/d/yy at h:mm"); + assertFormatDateAndOrDateTime(true, false, "h PM, ddd mmm dd"); + assertFormatDateAndOrDateTime(true, false, "dd/mm/yyyy hh:mm"); + assertFormatDateAndOrDateTime(true, false, "yyyy-mm-dd hh:mm:ss.000"); + assertFormatDateAndOrDateTime(true, false, "hh:mm dd/mm/yyyy"); + + assertFormatDateAndOrDateTime(false, true, "yyyy-mm-dd"); + assertFormatDateAndOrDateTime(false, true, "mmmm d \\[dddd\\]"); + assertFormatDateAndOrDateTime(false, true, "mmm dd, yyyy"); + assertFormatDateAndOrDateTime(false, true, "d-mmm"); + assertFormatDateAndOrDateTime(false, true, "dd.mm.yyyy"); + + assertFormatDateAndOrDateTime(false, false, "yyyy"); + assertFormatDateAndOrDateTime(false, false, "mmm-yyyy"); + assertFormatDateAndOrDateTime(false, false, "hh"); + assertFormatDateAndOrDateTime(false, false, "hh:mm"); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private void assertFormatDateAndOrDateTime(boolean expectDateTime, boolean expectDate, String format) + { + if(XlsxFileToRows.isDateTimeFormat(format)) + { + assertTrue(expectDateTime, format + " was considered a dateTime, but wasn't expected to."); + assertFalse(expectDate, format + " was considered a dateTime, but was expected to be a date."); + } + else if(XlsxFileToRows.isDateFormat(format)) + { + assertFalse(expectDateTime, format + " was considered a date, but was expected to be a dateTime."); + assertTrue(expectDate, format + " was considered a date, but was expected to."); + } + else + { + assertFalse(expectDateTime, format + " was not considered a dateTime, but was expected to."); + assertFalse(expectDate, format + " was considered a date, but was expected to."); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java new file mode 100644 index 00000000..a54e08ef --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java @@ -0,0 +1,209 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for BulkLoadMappingSuggester + *******************************************************************************/ +class BulkLoadMappingSuggesterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleFlat() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY); + List headerRow = List.of("Id", "First Name", "lastname", "email", "homestate"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("FLAT", bulkLoadProfile.getLayout()); + assertNull(getFieldByName(bulkLoadProfile, "id")); + assertEquals(1, getFieldByName(bulkLoadProfile, "firstName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "lastName").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "email").getColumnIndex()); + assertEquals(4, getFieldByName(bulkLoadProfile, "homeStateId").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleTall() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "shipto name", "sku", "quantity"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("TALL", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTallWithTableNamesOnAssociations() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("TALL", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testChallengingAddress1And2() + { + try + { + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address1", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address1", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + } + finally + { + reInitInstanceInContext(TestUtils.defineInstance()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleWide() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("WIDE", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku,0").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity,0").getColumnIndex()); + assertEquals(4, getFieldByName(bulkLoadProfile, "orderLine.sku,1").getColumnIndex()); + assertEquals(5, getFieldByName(bulkLoadProfile, "orderLine.quantity,1").getColumnIndex()); + + ///////////////////////////////////////////////////////////////// + // assert that the order of fields matches the file's ordering // + ///////////////////////////////////////////////////////////////// + assertEquals(List.of("orderNo", "shipToName", "orderLine.sku,0", "orderLine.quantity,0", "orderLine.sku,1", "orderLine.quantity,1"), + bulkLoadProfile.getFieldList().stream().map(f -> f.getFieldName()).toList()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private BulkLoadProfileField getFieldByName(BulkLoadProfile bulkLoadProfile, String fieldName) + { + return (bulkLoadProfile.getFieldList().stream() + .filter(f -> f.getFieldName().equals(fieldName)) + .findFirst().orElse(null)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java new file mode 100644 index 00000000..0cfe19fd --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -0,0 +1,185 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for ValueMapper + *******************************************************************************/ +class BulkLoadValueMapperTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( + "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), + "shipToName", Map.of("HoJu", "Homer", "Bart", "Bartholomew"), + "orderLine.sku", Map.of("ABC", "Alphabet"), + "orderLine.extrinsics.value", Map.of("foo", "bar", "bar", "baz"), + "extrinsics.key", Map.of("1", "one", "2", "two") + )); + + QRecord inputRecord = new QRecord() + .withValue("storeId", "QQQMart") + .withValue("shipToName", "HoJu") + .withAssociatedRecord("orderLine", new QRecord() + .withValue("sku", "ABC") + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "myKey") + .withValue("value", "foo") + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "yourKey") + .withValue("value", "bar") + ) + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", 1) + .withValue("value", "foo") + ); + JSONObject beforeJson = recordToJson(inputRecord); + + QRecord expectedRecord = new QRecord() + .withValue("storeId", 1) + .withValue("shipToName", "Homer") + .withAssociatedRecord("orderLine", new QRecord() + .withValue("sku", "Alphabet") + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "myKey") + .withValue("value", "bar") + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "yourKey") + .withValue("value", "baz") + ) + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "one") + .withValue("value", "foo") + ); + JSONObject expectedJson = recordToJson(expectedRecord); + + BulkLoadValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); + JSONObject actualJson = recordToJson(inputRecord); + + System.out.println("Before"); + System.out.println(beforeJson.toString(3)); + System.out.println("Actual"); + System.out.println(actualJson.toString(3)); + System.out.println("Expected"); + System.out.println(expectedJson.toString(3)); + + assertThat(actualJson).usingRecursiveComparison().isEqualTo(expectedJson); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + void testPossibleValue(Serializable inputValue, Serializable expectedValue, boolean expectErrors) throws QException + { + QRecord inputRecord = new QRecord().withValue("homeStateId", inputValue); + BulkLoadValueMapper.valueMapping(List.of(inputRecord), new BulkInsertMapping(), QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)); + assertEquals(expectedValue, inputRecord.getValue("homeStateId")); + + if(expectErrors) + { + assertThat(inputRecord.getErrors().get(0)).isInstanceOf(BulkLoadPossibleValueError.class); + } + else + { + assertThat(inputRecord.getErrors()).isNullOrEmpty(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValues() throws QException + { + testPossibleValue(1, 1, false); + testPossibleValue("1", 1, false); + testPossibleValue("1.0", 1, false); + testPossibleValue(new BigDecimal("1.0"), 1, false); + testPossibleValue("IL", 1, false); + testPossibleValue("il", 1, false); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // unmappables - should have a null value (this used to not be the case - the bad-value would come through...) // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + testPossibleValue(512, null, true); // an int, but not in the PVS + testPossibleValue("USA", null, true); + testPossibleValue(true, null, true); + testPossibleValue(new BigDecimal("4.7"), null, true); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static JSONObject recordToJson(QRecord record) + { + JSONObject jsonObject = new JSONObject(); + for(Map.Entry valueEntry : CollectionUtils.nonNullMap(record.getValues()).entrySet()) + { + jsonObject.put(valueEntry.getKey(), valueEntry.getValue()); + } + for(Map.Entry> associationEntry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + JSONArray jsonArray = new JSONArray(); + for(QRecord associationRecord : CollectionUtils.nonNullList(associationEntry.getValue())) + { + jsonArray.put(recordToJson(associationRecord)); + } + jsonObject.put(associationEntry.getKey(), jsonArray); + } + return (jsonObject); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java new file mode 100644 index 00000000..485c5e54 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -0,0 +1,262 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FlatRowsToRecord + *******************************************************************************/ +class FlatRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToHeaderNameMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { "id", "firstName", "Last Name", "Ignore", "cost" }, + new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, + new Serializable[] { 2, "Marge", "Simpson", false, "" }, + new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" }, + new Serializable[] { "", "", "", "", "" } // all blank row (we can get these at the bottoms of files) - make sure it doesn't become a record. + )); + + BulkLoadFileRow header = fileToRows.next(); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name", + "cost", "cost" + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00")))) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(1, records.size()); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); + assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 2", records.get(0).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(2, records.size()); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 3", records.get(0).getBackendDetail("rowNos")); + assertEquals("Row 4", records.get(1).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + assertEquals("Row 5", records.get(0).getBackendDetail("rowNos")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToColumnIndexMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + // 0, 1, 2, 3, 4 + new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, + new Serializable[] { 2, "Marge", "Simpson", false, "" }, + new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" } + )); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "firstName", 1, + "lastName", 2, + "cost", 4 + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00")))) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, null, mapping, 1); + assertEquals(1, records.size()); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); + assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 1", records.get(0).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, null, mapping, 2); + assertEquals(2, records.size()); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 2", records.get(0).getBackendDetail("rowNos")); + assertEquals("Row 3", records.get(1).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, null, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + assertEquals("Row 4", records.get(0).getBackendDetail("rowNos")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToIndexMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { 1, "Homer", "Simpson", true }, + new Serializable[] { 2, "Marge", "Simpson", false }, + new Serializable[] { 3, "Bart", "Simpson", "A" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1 } + )); + + BulkLoadFileRow header = null; + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "firstName", 1, + "lastName", 2 + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(3, records.get(0).getValues().size()); // make sure no additional values were set + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValueMappings() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { "id", "firstName", "Last Name", "Home State" }, + new Serializable[] { 1, "Homer", "Simpson", 1 }, + new Serializable[] { 2, "Marge", "Simpson", "MO" }, + new Serializable[] { 3, "Bart", "Simpson", null }, + new Serializable[] { 4, "Ned", "Flanders", "Not a state" }, + new Serializable[] { 5, "Mr.", "Burns", 5 } + )); + + BulkLoadFileRow header = fileToRows.next(); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name", + "homeStateId", "Home State" + )) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(5, records.size()); + assertEquals(List.of("Homer", "Marge", "Bart", "Ned", "Mr."), getValues(records, "firstName")); + assertEquals(ListBuilder.of(1, 2, null, null, null), getValues(records, "homeStateId")); + + assertThat(records.get(0).getErrors()).isNullOrEmpty(); + assertThat(records.get(1).getErrors()).isNullOrEmpty(); + assertThat(records.get(2).getErrors()).isNullOrEmpty(); + assertThat(records.get(3).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option")); + assertThat(records.get(4).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java new file mode 100644 index 00000000..b59f3b1d --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -0,0 +1,599 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for TallRowsToRecord + *******************************************************************************/ +class TallRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLines() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity + 1, Homer, Simpson, DONUT, 12 + , Homer, Simpson, BEER, 500 + , Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + , Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 2-4", order.getBackendDetail("rowNos")); + assertEquals(3, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 4", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 5-6", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + + + /******************************************************************************* + ** test to show that we can do 1 default line item (child record) for each + ** header record. + *******************************************************************************/ + @Test + void testOrderAndLinesWithLineValuesFromDefaults() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName + 1, Homer, Simpson + 2, Ned, Flanders + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.sku", "NUCLEAR-ROD", + "orderLine.quantity", 1 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(1, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(1, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Row 3", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutHeader() throws QException + { + // 0, 1, 2, 3, 4 + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + 1, Homer, Simpson, DONUT, 12 + , Homer, Simpson, BEER, 500 + , Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + , Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = null; + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "orderNo", 0, + "shipToName", 1, + "orderLine.sku", 3, + "orderLine.quantity", 4 + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 1-3", order.getBackendDetail("rowNos")); + assertEquals(3, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 4-5", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsic() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart + 1, , , BEER, 500, Coupon Code, 10QOff + 1, , , COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + 2, , , LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(0), + "orderLine", List.of(3), + "extrinsics", List.of(5) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(2, order.getAssociatedRecords().get("extrinsics").size()); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException + { + Integer defaultStoreId = 101; + String defaultLineNo = "102"; + String defaultOrderLineExtraSource = "file"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate + 1, , , DONUT, , Coupon Code, 10QOff, Size, Large + 1, , , BEER, 500, , , Flavor, Hops + 1, , , COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, , , Flavor, King James + 2, , , LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.source", defaultOrderLineExtraSource + )) + .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(0), + "orderLine", List.of(3), + "extrinsics", List.of(5), + "orderLine.extrinsics", List.of(7) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAutomaticGroupByAllIndexes() throws QException + { + Integer defaultStoreId = 101; + String defaultLineNo = "102"; + String defaultOrderLineExtraSource = "file"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate + 1, Homer, Simpson, DONUT, 12, Coupon Code, 10QOff, Size, Large + 1, Homer, Simpson, BEER, 500, , , Flavor, Hops + 1, Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, , , Flavor, King James + 2, Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.source", defaultOrderLineExtraSource + )) + .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSingleLine() throws QException + { + Integer defaultStoreId = 101; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName + 1, Homer, Simpson + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPagination() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity + 1, Homer, Simpson, DONUT, 12 + 2, Ned, Flanders, BIBLE, 7 + 2, Ned, Flanders, LAWNMOWER, 1 + 3, Bart, Simpson, SKATEBOARD,1 + 3, Bart, Simpson, SLINGSHOT, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Rows 3-4", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(1, records.size()); + order = records.get(0); + assertEquals(3, order.getValueInteger("orderNo")); + assertEquals("Bart", order.getValueString("shipToName")); + assertEquals(List.of("SKATEBOARD", "SLINGSHOT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Rows 5-6", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testShouldProcessAssociation() + { + TallRowsToRecord tallRowsToRecord = new TallRowsToRecord(); + assertTrue(tallRowsToRecord.shouldProcessAssociation(null, "foo")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("", "foo")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("foo.bar", "foo.bar.baz")); + + assertFalse(tallRowsToRecord.shouldProcessAssociation(null, "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz.biz", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar.baz")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java new file mode 100644 index 00000000..3a8677bb --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -0,0 +1,291 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping + *******************************************************************************/ +class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, two + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku,0", "SKU 1", + "orderLine.quantity,0", "Quantity 1", + "orderLine.sku,1", "SKU 2", + "orderLine.quantity,1", "Quantity 2", + "orderLine.sku,2", "SKU 3", + "orderLine.quantity,2", "Quantity 3" + )) + .withMappedAssociations(List.of("orderLine")) + .withFieldNameToValueMapping(Map.of( + "orderLine.quantity,2", Map.of("two", 2) + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 2), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithLineValuesFromDefaults() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1 + 1, Homer, Simpson, DONUT, 12, + 2, Ned, Flanders, + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku,0", "SKU 1", + "orderLine.quantity,0", "Quantity 1" + )) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.sku,1", "NUCLEAR-ROD", + "orderLine.quantity,1", 1 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutHeader() throws QException + { + // 0, 1, 2, 3, 4, 5, 6, 7, 8 + String csv = """ + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = null; + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "orderNo", 0, + "shipToName", 1, + "orderLine.sku,0", 3, + "orderLine.quantity,0", 4, + "orderLine.sku,1", 5, + "orderLine.quantity,1", 6, + "orderLine.sku,2", 7, + "orderLine.quantity,2", 8 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(MapBuilder.of(() -> new HashMap()) + .with("orderNo", "orderNo") + .with("shipToName", "Ship To") + + .with("orderLine.sku,0", "SKU 1") + .with("orderLine.quantity,0", "Quantity 1") + .with("orderLine.sku,1", "SKU 2") + .with("orderLine.quantity,1", "Quantity 2") + .with("orderLine.sku,2", "SKU 3") + .with("orderLine.quantity,2", "Quantity 3") + + .with("extrinsics.key,0", "Extrinsic Key 1") + .with("extrinsics.value,0", "Extrinsic Value 1") + .with("extrinsics.key,1", "Extrinsic Key 2") + .with("extrinsics.value,1", "Extrinsic Value 2") + .build() + ) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.lineNumber,0", "1", + "orderLine.lineNumber,1", "2", + "orderLine.lineNumber,2", "3" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("1", "2", "3"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("1", "2"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java new file mode 100644 index 00000000..8bd49beb --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -0,0 +1,306 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecord + *******************************************************************************/ +class WideRowsToRecordWithSpreadMappingTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + testOrderAndLines(""" + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithDupes() throws QException + { + testOrderAndLines(""" + orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void testOrderAndLines(String csv) throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + testOrderLinesAndOrderExtrinsic(""" + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithDupes() throws QException + { + testOrderLinesAndOrderExtrinsic(""" + orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testOrderLinesAndOrderExtrinsic(String csv) throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException + { + testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(""" + orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2, Line Extrinsic Value 2, SKU 2, Quantity 2, Line Extrinsic Key 1, Line Extrinsic Value 1, SKU 3, Quantity 3, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2 + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithDupes() throws QException + { + testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(""" + orderNo, Ship To, lastName, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException + { + Integer defaultStoreId = 42; + String defaultLineNo = "47"; + String defaultLineExtraValue = "bar"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.value", defaultLineExtraValue + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(2); + assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java new file mode 100644 index 00000000..59a25481 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java @@ -0,0 +1,50 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for NoopLoadStep + *******************************************************************************/ +class NoopLoadStepTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ////////////////////////////////////// + // sorry, just here for coverage... // + ////////////////////////////////////// + new NoopLoadStep().runOnePage(new RunBackendStepInput(), new RunBackendStepOutput()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java new file mode 100644 index 00000000..a1692327 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java @@ -0,0 +1,186 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfileMetaDataProvider; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit tests for all saved-bulk-load-profile processes + *******************************************************************************/ +class SavedBulkLoadProfileProcessTests extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QInstance qInstance = QContext.getQInstance(); + new SavedBulkLoadProfileMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + { + //////////////////////////////////////////// + // query - should be no profiles to start // + //////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertEquals(0, ((List) runProcessOutput.getValues().get("savedBulkLoadProfileList")).size()); + } + + Integer savedBulkLoadProfileId; + { + ///////////////////////// + // store a new profile // + ///////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("label", "My Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + savedBulkLoadProfileId = savedBulkLoadProfileList.get(0).getValueInteger("id"); + assertNotNull(savedBulkLoadProfileId); + + ////////////////////////////////////////////////////////////////// + // try to store it again - should throw a "duplicate" exception // + ////////////////////////////////////////////////////////////////// + assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput)) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("already have a saved Bulk Load Profile"); + } + + { + ////////////////////////////////////// + // query - should find our profiles // + ////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + assertEquals(1, savedBulkLoadProfileList.get(0).getValueInteger("id")); + assertEquals("My Profile", savedBulkLoadProfileList.get(0).getValueString("label")); + } + + { + //////////////////////// + // update our Profile // + //////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", savedBulkLoadProfileId); + runProcessInput.addValue("label", "My Updated Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + assertEquals(1, savedBulkLoadProfileList.get(0).getValueInteger("id")); + assertEquals("My Updated Profile", savedBulkLoadProfileList.get(0).getValueString("label")); + } + + Integer anotherSavedProfileId; + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // store a second one w/ different name (will be used below in update-dupe-check use-case) // + ///////////////////////////////////////////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("label", "My Second Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + anotherSavedProfileId = savedBulkLoadProfileList.get(0).getValueInteger("id"); + } + + { + ///////////////////////////////////////////////// + // try to rename the second to match the first // + ///////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", anotherSavedProfileId); + runProcessInput.addValue("label", "My Updated Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + + ////////////////////////////////////////// + // should throw a "duplicate" exception // + ////////////////////////////////////////// + assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput)) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("already have a saved Bulk Load Profile"); + } + + { + ///////////////////////// + // delete our profiles // + ///////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(DeleteSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", savedBulkLoadProfileId); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + + runProcessInput.addValue("id", anotherSavedProfileId); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + } + + { + ///////////////////////////////////////// + // query - should be no profiles again // + ///////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertEquals(0, ((List) runProcessOutput.getValues().get("savedBulkLoadProfileList")).size()); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java index cbe594f6..b177b9bd 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java @@ -25,13 +25,17 @@ package com.kingsrook.qqq.backend.core.processes.locks; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -39,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QUser; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -227,6 +232,28 @@ class ProcessLockUtilsTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testReleaseBadInputs() + { + ////////////////////////////////////////////////////////// + // make sure we don't blow up, just noop in these cases // + ////////////////////////////////////////////////////////// + QCollectingLogger qCollectingLogger = QLogger.activateCollectingLoggerForClass(ProcessLockUtils.class); + ProcessLockUtils.releaseById(null); + ProcessLockUtils.release(null); + ProcessLockUtils.releaseMany(null); + ProcessLockUtils.releaseByIds(null); + ProcessLockUtils.releaseMany(ListBuilder.of(null)); + ProcessLockUtils.releaseByIds(ListBuilder.of(null)); + QLogger.deactivateCollectingLoggerForClass(ProcessLockUtils.class); + assertEquals(6, qCollectingLogger.getCollectedMessages().stream().filter(m -> m.getMessage().contains("noop")).count()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -304,7 +331,7 @@ class ProcessLockUtilsTest extends BaseTest ////////////////////////////////////////////// // checkin w/ a time - sets it to that time // ////////////////////////////////////////////// - Instant specifiedTime = Instant.now(); + Instant specifiedTime = Instant.now().plusSeconds(47); ProcessLockUtils.checkIn(processLock, specifiedTime); processLock = ProcessLockUtils.getById(processLock.getId()); assertEquals(specifiedTime, processLock.getExpiresAtTimestamp()); @@ -380,4 +407,122 @@ class ProcessLockUtilsTest extends BaseTest assertNull(processLock.getExpiresAtTimestamp()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMany() throws QException + { + ///////////////////////////////////////////////// + // make sure that we can create multiple locks // + ///////////////////////////////////////////////// + List keys = List.of("1", "2", "3"); + List processLocks = new ArrayList<>(); + Map results = ProcessLockUtils.createMany(keys, "typeA", "me"); + for(String key : keys) + { + ProcessLock processLock = results.get(key).processLock(); + assertNotNull(processLock.getId()); + assertNotNull(processLock.getCheckInTimestamp()); + assertNull(processLock.getExpiresAtTimestamp()); + processLocks.add(processLock); + } + + ///////////////////////////////////////////////////////// + // make sure we can't create a second for the same key // + ///////////////////////////////////////////////////////// + assertThatThrownBy(() -> ProcessLockUtils.create("1", "typeA", "you")) + .isInstanceOf(UnableToObtainProcessLockException.class) + .hasMessageContaining("Held by: " + QContext.getQSession().getUser().getIdReference()) + .hasMessageContaining("with details: me") + .hasMessageNotContaining("expiring at: 20") + .matches(e -> ((UnableToObtainProcessLockException) e).getExistingLock() != null); + + ///////////////////////////////////////////////////////// + // make sure we can create another for a different key // + ///////////////////////////////////////////////////////// + ProcessLockUtils.create("4", "typeA", "him"); + + ///////////////////////////////////////////////////////////////////// + // make sure we can create another for a different type (same key) // + ///////////////////////////////////////////////////////////////////// + ProcessLockUtils.create("1", "typeB", "her"); + + //////////////////////////////////////////////////////////////////// + // now try to create some that will overlap, but one that'll work // + //////////////////////////////////////////////////////////////////// + keys = List.of("3", "4", "5"); + results = ProcessLockUtils.createMany(keys, "typeA", "me"); + for(String key : List.of("3", "4")) + { + UnableToObtainProcessLockException exception = results.get(key).unableToObtainProcessLockException(); + assertNotNull(exception); + } + + ProcessLock processLock = results.get("5").processLock(); + assertNotNull(processLock.getId()); + assertNotNull(processLock.getCheckInTimestamp()); + assertNull(processLock.getExpiresAtTimestamp()); + processLocks.add(processLock); + + ////////////////////////////// + // make sure we can release // + ////////////////////////////// + ProcessLockUtils.releaseMany(processLocks); + + /////////////////////////////////////////////////////// + // make sure can re-lock 1 now after it was released // + /////////////////////////////////////////////////////// + processLock = ProcessLockUtils.create("1", "typeA", "you"); + assertNotNull(processLock.getId()); + assertEquals("you", processLock.getDetails()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testManyWithSleep() throws QException + { + ///////////////////////////////////////////////// + // make sure that we can create multiple locks // + ///////////////////////////////////////////////// + List keys = List.of("1", "2", "3"); + Map results0 = ProcessLockUtils.createMany(keys, "typeB", "me"); + for(String key : keys) + { + assertNotNull(results0.get(key).processLock()); + } + + //////////////////////////////////////////////////////////// + // try again - and 2 and 3 should fail, if we don't sleep // + //////////////////////////////////////////////////////////// + keys = List.of("2", "3", "4"); + Map results1 = ProcessLockUtils.createMany(keys, "typeB", "you"); + assertNull(results1.get("2").processLock()); + assertNull(results1.get("3").processLock()); + assertNotNull(results1.get("4").processLock()); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // try another insert, which should initially succeed for #5, then sleep, and eventually succeed on 3 & 4 as well // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + keys = List.of("3", "4", "5"); + Map results2 = ProcessLockUtils.createMany(keys, "typeB", "them", Duration.of(1, ChronoUnit.SECONDS), Duration.of(3, ChronoUnit.SECONDS)); + for(String key : keys) + { + assertNotNull(results2.get(key).processLock()); + } + + //////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that we have a different ids for some that expired and then succeeded post-sleep // + //////////////////////////////////////////////////////////////////////////////////////////////// + assertNotEquals(results0.get("3").processLock().getId(), results2.get("3").processLock().getId()); + assertNotEquals(results1.get("4").processLock().getId(), results2.get("4").processLock().getId()); + + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/tracing/LoggingProcessTracerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/tracing/LoggingProcessTracerTest.java new file mode 100644 index 00000000..ec3c8e60 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/tracing/LoggingProcessTracerTest.java @@ -0,0 +1,93 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.utils.CollectionAssert; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for LoggingProcessTracer + *******************************************************************************/ +class LoggingProcessTracerTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QLogger.deactivateCollectingLoggerForClass(LoggingProcessTracer.class); + } + + + + /******************************************************************************* + ** this test is based on RunProcessTest#testBreakOnFrontendSteps + *******************************************************************************/ + @Test + void test() throws QException + { + ///////////////////////////////////////////////////// + // activate the tracer for this run of the process // + ///////////////////////////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.addValue(RunProcessAction.PROCESS_TRACER_CODE_REFERENCE_FIELD, new QCodeReference(LoggingProcessTracer.class)); + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(LoggingProcessTracer.class); + + input.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + input.setCallback(QProcessCallbackFactory.forRecord(new QRecord().withValue("id", 1))); + RunProcessOutput result0 = new RunProcessAction().execute(input); + assertNotNull(result0); + + CollectionAssert.assertThat(collectingLogger.getCollectedMessages()) + .matchesAll(List.of("Starting process", "Breaking process"), + (s, clm) -> clm.getMessageAsJSONObject().getString("message").equals(s)); + collectingLogger.clear(); + + /////////////////////////////////////////////////// + // now re-run (resume) to the end of the process // + /////////////////////////////////////////////////// + input.setStartAfterStep(result0.getProcessState().getNextStepName().get()); + RunProcessOutput result1 = new RunProcessAction().execute(input); + CollectionAssert.assertThat(collectingLogger.getCollectedMessages()) + .matchesAll(List.of("Resuming process", "Starting process step", "Finished process step", "Finished process"), + (s, clm) -> clm.getMessageAsJSONObject().getString("message").equals(s)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/tracing/NoopProcessTracerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/tracing/NoopProcessTracerTest.java new file mode 100644 index 00000000..b46592aa --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/tracing/NoopProcessTracerTest.java @@ -0,0 +1,61 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for NoopProcessTracer ... kinda a BS test, but here to prevent + ** a missing-class for code coverage... + *******************************************************************************/ +class NoopProcessTracerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ////////////////////////////////////////////////////////// + // activate the noop tracer for this run of the process // + ////////////////////////////////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.addValue(RunProcessAction.PROCESS_TRACER_CODE_REFERENCE_FIELD, new QCodeReference(NoopProcessTracer.class)); + + input.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + input.setCallback(QProcessCallbackFactory.forRecord(new QRecord().withValue("id", 1))); + new RunProcessAction().execute(input); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java new file mode 100644 index 00000000..3b259ce9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.text.ParseException; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for CronDescriber + *******************************************************************************/ +class CronDescriberTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws ParseException + { + assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * * * ?")); + assertEquals("At 0 seconds, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 * * * * ?")); + assertEquals("At 0 seconds, 0 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 * * * ?")); + assertEquals("At 0 seconds, 0 and 30 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0,30 * * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 0 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 1 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 1 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 11 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 11 * * ?")); + assertEquals("At 0 seconds, 0 minutes, noon, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 12 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 1 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 13 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 11 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 23 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on day 10 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10 * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on days 10 and 20 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10,20 * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on days from 10 to 15 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10-15 * ?")); + assertEquals("At from 10 to 15 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("10-15 0 0 * * ?")); + assertEquals("At 30 seconds, 30 minutes, from 8 AM to 4 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("30 30 8-16 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every 3 days starting at 0 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 */3 * ?")); + assertEquals("At every 5 seconds starting at 0, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0/5 0 0 * * ?")); + assertEquals("At 0 seconds, every 30 minutes starting at 3, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 3/30 0 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Monday, Wednesday, and Friday.", CronDescriber.getDescription("0 0 0 * * MON,WED,FRI")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, from Monday to Friday.", CronDescriber.getDescription("0 0 0 * * MON-FRI")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday and Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); + assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, and 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); + assertEquals("At every 5 seconds starting at 0, 14, 18, 3-39, and 52 minutes, every hour, on every day of January, March, and September, from Monday to Friday, in 2002-2010.", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + + assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * ? * *")); + assertEquals("At every second, every minute, every hour, on every day of January to June, every day of the week.", CronDescriber.getDescription("* * * ? 1-6 *")); + assertEquals("At every second, every minute, every hour, on days 1, 3, and 5 of every month, every day of the week.", CronDescriber.getDescription("* * * 1,3,5 * *")); + // todo fix has 2-4 hours and 3 PM, s/b 2 AM to 4 AM and 3 PM assertEquals("At every second, every minute, every hour, on days 1, 3, and 5 of every month, every day of the week.", CronDescriber.getDescription("* * 2-4,15 1,3,5 * *")); + // hour failing on 3,2-7 (at least in TS side?) + // 3,2-7 makes 3,2 to July + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java new file mode 100644 index 00000000..26dc8879 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for CronExpressionTooltipFieldBehavior + *******************************************************************************/ +class CronExpressionTooltipFieldBehaviorTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QFieldMetaData field = new QFieldMetaData("cronExpression", QFieldType.STRING); + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE) + .addField(field); + + CronExpressionTooltipFieldBehavior.addToField(field); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SHAPE).withRecord( + new QRecord().withValue("name", "Square").withValue("cronExpression", "* * * * * ?"))); + + QRecord record = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_SHAPE).withPrimaryKey(1).withShouldGenerateDisplayValues(true)); + assertThat(record.getDisplayValue("cronExpression:" + AdornmentType.TooltipValues.TOOLTIP_DYNAMIC)) + .contains("every second"); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionAssert.java new file mode 100644 index 00000000..d5fc09a9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/CollectionAssert.java @@ -0,0 +1,184 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils; + + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.BiFunction; +import org.assertj.core.api.AbstractAssert; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** assertions against collections + *******************************************************************************/ +public class CollectionAssert extends AbstractAssert, Collection> +{ + /*************************************************************************** + ** + ***************************************************************************/ + protected CollectionAssert(Collection actual, Class selfType) + { + super(actual, selfType); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static CollectionAssert assertThat(Collection actualCollection) + { + return (new CollectionAssert<>(actualCollection, CollectionAssert.class)); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public CollectionAssert matchesAllowingExpectedToHaveMore(Collection expected, BiFunction predicate) + { + if(actual == null && expected != null) + { + fail("Actual collection was null, but expected collection was not-null"); + return (this); + } + else if(actual != null && expected == null) + { + fail("Actual collection was not null, but expected collection was null"); + return (this); + } + else if(actual == null && expected == null) + { + return (this); + } + + assertTrue(actual.size() >= expected.size(), "Actual collection size [" + actual.size() + "] should be >= expected collection size [" + expected.size() + "]"); + + matchElements(expected, predicate); + + return (this); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private void matchElements(Collection expected, BiFunction predicate) + { + List nonMatchingExpectedIndexes = new ArrayList<>(); + Set matchedActualIndexes = new HashSet<>(); + + List expectedList = new ArrayList<>(expected); + List actualList = new ArrayList<>(actual); + + for(int eIndex = 0; eIndex < expectedList.size(); eIndex++) + { + E e = expectedList.get(eIndex); + + boolean matchedThieE = false; + + for(int aIndex = 0; aIndex < actualList.size(); aIndex++) + { + A a = actualList.get(aIndex); + if(!matchedThieE && !matchedActualIndexes.contains(aIndex)) // don't re-check an already-matched item + { + if(predicate.apply(e, a)) + { + matchedActualIndexes.add(aIndex); + matchedThieE = true; + } + } + } + + if(!matchedThieE) + { + nonMatchingExpectedIndexes.add(eIndex); + } + } + + assertTrue(nonMatchingExpectedIndexes.isEmpty(), "Did not find a match for indexes " + nonMatchingExpectedIndexes + "\n from expected collection: " + expected + "\n in actual collection: " + actual); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public CollectionAssert matchesAll(Collection expected, BiFunction predicate) + { + if(actual == null && expected != null) + { + fail("Actual collection was null, but expected collection was not-null"); + return (this); + } + else if(actual != null && expected == null) + { + fail("Actual collection was not null, but expected collection was null"); + return (this); + } + else if(actual == null && expected == null) + { + return (this); + } + + assertEquals(expected.size(), actual.size(), "Expected size of collections"); + + matchElements(expected, predicate); + + return (this); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public CollectionAssert isNullOrEmpty() + { + if(actual != null) + { + assertEquals(0, actual.size(), "Expected collection to be null or empty"); + } + return (this); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public CollectionAssert isEmpty() + { + assertEquals(0, actual.size(), "Expected collection to be empty"); + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java index 0f0f15ec..5b27cb3b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java @@ -105,6 +105,7 @@ class StringUtilsTest extends BaseTest assertEquals("Foo bar", StringUtils.allCapsToMixedCase("FOo bar")); assertEquals("Foo Bar", StringUtils.allCapsToMixedCase("FOo BAr")); assertEquals("foo bar", StringUtils.allCapsToMixedCase("foo bar")); + assertEquals("Foo Bar", StringUtils.allCapsToMixedCase("FOO_BAR")); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index ff274dc9..54502074 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -70,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; @@ -142,6 +143,7 @@ public class TestUtils public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; public static final String TABLE_NAME_TWO_KEYS = "twoKeys"; + public static final String TABLE_NAME_MEMORY_STORAGE = "memoryStorage"; public static final String TABLE_NAME_PERSON = "person"; public static final String TABLE_NAME_SHAPE = "shape"; public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache"; @@ -204,6 +206,7 @@ public class TestUtils qInstance.addTable(defineTablePerson()); qInstance.addTable(defineTableTwoKeys()); + qInstance.addTable(defineTableMemoryStorage()); qInstance.addTable(definePersonFileTable()); qInstance.addTable(definePersonMemoryTable()); qInstance.addTable(definePersonMemoryCacheTable()); @@ -594,6 +597,22 @@ public class TestUtils + /******************************************************************************* + ** Define a table in the memory store that can be used for the StorageAction + *******************************************************************************/ + public static QTableMetaData defineTableMemoryStorage() + { + return new QTableMetaData() + .withName(TABLE_NAME_MEMORY_STORAGE) + .withLabel("Memory Storage") + .withBackendName(MEMORY_BACKEND_NAME) + .withPrimaryKeyField("reference") + .withField(new QFieldMetaData("reference", QFieldType.STRING).withIsEditable(false)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB)); + } + + + /******************************************************************************* ** Define the 'person' table used in standard tests. *******************************************************************************/ @@ -644,6 +663,7 @@ public class TestUtils .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderNo", QFieldType.STRING)) + .withField(new QFieldMetaData("shipToName", QFieldType.STRING).withMaxLength(200).withBehavior(ValueTooLongBehavior.ERROR)) .withField(new QFieldMetaData("orderDate", QFieldType.DATE)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER)) .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock() @@ -700,7 +720,8 @@ public class TestUtils .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("lineItemId", QFieldType.INTEGER)) .withField(new QFieldMetaData("key", QFieldType.STRING)) - .withField(new QFieldMetaData("value", QFieldType.STRING)); + .withField(new QFieldMetaData("value", QFieldType.STRING)) + .withField(new QFieldMetaData("source", QFieldType.STRING)); // doesn't really make sense, but useful to have an extra field here in some bulk-load tests } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java index d61a56c1..ee1f69f1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java @@ -251,6 +251,9 @@ class ValueUtilsTest extends BaseTest assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a,b")); assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("1980/05/31")); assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new Object())).getMessage()).contains("Unsupported class"); + + expected = Instant.parse("1980-05-31T01:30:00Z"); + assertEquals(expected, ValueUtils.getValueAsInstant("1980-05-31 1:30:00")); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java index 4dff56c6..daddaaf2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java @@ -91,6 +91,57 @@ class AggregatesTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLong() + { + LongAggregates aggregates = new LongAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + aggregates.add(5L); + assertEquals(1, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(5, aggregates.getMax()); + assertEquals(5, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("5"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(10L); + assertEquals(2, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(10, aggregates.getMax()); + assertEquals(15, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("7.5"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(15L); + assertEquals(3, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(15, aggregates.getMax()); + assertEquals(30, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(null); + assertEquals(3, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(15, aggregates.getMax()); + assertEquals(30, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("750"), aggregates.getProduct()); + assertEquals(new BigDecimal("25.0000"), aggregates.getVariance()); + assertEquals(new BigDecimal("5.0000"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("16.6667"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("4.0824"), Offset.offset(new BigDecimal(".0001"))); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java new file mode 100644 index 00000000..63919551 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java @@ -0,0 +1,52 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for CaseInsensitiveKeyMap + *******************************************************************************/ +class CaseInsensitiveKeyMapTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + CaseInsensitiveKeyMap map = new CaseInsensitiveKeyMap<>(); + map.put("One", 1); + map.put("one", 1); + map.put("ONE", 1); + assertEquals(1, map.get("one")); + assertEquals(1, map.get("One")); + assertEquals(1, map.get("oNe")); + assertEquals(1, map.size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java new file mode 100644 index 00000000..594aeabc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java @@ -0,0 +1,201 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.math.BigDecimal; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for TransformedKeyMap + *******************************************************************************/ +@SuppressWarnings({ "RedundantCollectionOperation", "RedundantOperationOnEmptyContainer" }) +class TransformedKeyMapTest extends BaseTest +{ + private static final BigDecimal BIG_DECIMAL_TWO = BigDecimal.valueOf(2); + private static final BigDecimal BIG_DECIMAL_THREE = BigDecimal.valueOf(3); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCaseInsensitiveKeyMap() + { + TransformedKeyMap caseInsensitiveKeys = new TransformedKeyMap<>(key -> key.toLowerCase()); + caseInsensitiveKeys.put("One", 1); + caseInsensitiveKeys.put("one", 1); + caseInsensitiveKeys.put("ONE", 1); + assertEquals(1, caseInsensitiveKeys.get("one")); + assertEquals(1, caseInsensitiveKeys.get("One")); + assertEquals(1, caseInsensitiveKeys.get("oNe")); + assertEquals(1, caseInsensitiveKeys.size()); + + ////////////////////////////////////////////////// + // get back the first way it was put in the map // + ////////////////////////////////////////////////// + assertEquals("One", caseInsensitiveKeys.entrySet().iterator().next().getKey()); + assertEquals("One", caseInsensitiveKeys.keySet().iterator().next()); + + assertEquals(1, caseInsensitiveKeys.entrySet().size()); + assertEquals(1, caseInsensitiveKeys.keySet().size()); + + for(String key : caseInsensitiveKeys.keySet()) + { + assertEquals(1, caseInsensitiveKeys.get(key)); + } + + for(Map.Entry entry : caseInsensitiveKeys.entrySet()) + { + assertEquals("One", entry.getKey()); + assertEquals(1, entry.getValue()); + } + + ///////////////////////////// + // add a second unique key // + ///////////////////////////// + caseInsensitiveKeys.put("Two", 2); + assertEquals(2, caseInsensitiveKeys.size()); + assertEquals(2, caseInsensitiveKeys.entrySet().size()); + assertEquals(2, caseInsensitiveKeys.keySet().size()); + + //////////////////////////////////////// + // make sure remove works as expected // + //////////////////////////////////////// + caseInsensitiveKeys.remove("TWO"); + assertNull(caseInsensitiveKeys.get("Two")); + assertNull(caseInsensitiveKeys.get("two")); + assertEquals(1, caseInsensitiveKeys.size()); + assertEquals(1, caseInsensitiveKeys.keySet().size()); + assertEquals(1, caseInsensitiveKeys.entrySet().size()); + + /////////////////////////////////////// + // make sure clear works as expected // + /////////////////////////////////////// + caseInsensitiveKeys.clear(); + assertNull(caseInsensitiveKeys.get("one")); + assertEquals(0, caseInsensitiveKeys.size()); + assertEquals(0, caseInsensitiveKeys.keySet().size()); + assertEquals(0, caseInsensitiveKeys.entrySet().size()); + + ///////////////////////////////////////// + // make sure put-all works as expected // + ///////////////////////////////////////// + caseInsensitiveKeys.putAll(Map.of("One", 1, "one", 1, "ONE", 1, "TwO", 2, "tWo", 2, "three", 3)); + assertEquals(1, caseInsensitiveKeys.get("oNe")); + assertEquals(2, caseInsensitiveKeys.get("two")); + assertEquals(3, caseInsensitiveKeys.get("Three")); + assertEquals(3, caseInsensitiveKeys.size()); + assertEquals(3, caseInsensitiveKeys.entrySet().size()); + assertEquals(3, caseInsensitiveKeys.keySet().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStringToNumberMap() + { + TransformedKeyMap multiLingualWordToNumber = new TransformedKeyMap<>(key -> switch(key.toLowerCase()) + { + case "one", "uno", "eins" -> 1; + case "two", "dos", "zwei" -> 2; + case "three", "tres", "drei" -> 3; + default -> null; + }); + multiLingualWordToNumber.put("One", BigDecimal.ONE); + multiLingualWordToNumber.put("uno", BigDecimal.ONE); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("one")); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("uno")); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("eins")); + assertEquals(1, multiLingualWordToNumber.size()); + + ////////////////////////////////////////////////// + // get back the first way it was put in the map // + ////////////////////////////////////////////////// + assertEquals("One", multiLingualWordToNumber.entrySet().iterator().next().getKey()); + assertEquals("One", multiLingualWordToNumber.keySet().iterator().next()); + + assertEquals(1, multiLingualWordToNumber.entrySet().size()); + assertEquals(1, multiLingualWordToNumber.keySet().size()); + + for(String key : multiLingualWordToNumber.keySet()) + { + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get(key)); + } + + for(Map.Entry entry : multiLingualWordToNumber.entrySet()) + { + assertEquals("One", entry.getKey()); + assertEquals(BigDecimal.ONE, entry.getValue()); + } + + ///////////////////////////// + // add a second unique key // + ///////////////////////////// + multiLingualWordToNumber.put("Two", BIG_DECIMAL_TWO); + assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("Dos")); + assertEquals(2, multiLingualWordToNumber.size()); + assertEquals(2, multiLingualWordToNumber.entrySet().size()); + assertEquals(2, multiLingualWordToNumber.keySet().size()); + + //////////////////////////////////////// + // make sure remove works as expected // + //////////////////////////////////////// + multiLingualWordToNumber.remove("ZWEI"); + assertNull(multiLingualWordToNumber.get("Two")); + assertNull(multiLingualWordToNumber.get("Dos")); + assertEquals(1, multiLingualWordToNumber.size()); + assertEquals(1, multiLingualWordToNumber.keySet().size()); + assertEquals(1, multiLingualWordToNumber.entrySet().size()); + + /////////////////////////////////////// + // make sure clear works as expected // + /////////////////////////////////////// + multiLingualWordToNumber.clear(); + assertNull(multiLingualWordToNumber.get("eins")); + assertNull(multiLingualWordToNumber.get("One")); + assertEquals(0, multiLingualWordToNumber.size()); + assertEquals(0, multiLingualWordToNumber.keySet().size()); + assertEquals(0, multiLingualWordToNumber.entrySet().size()); + + ///////////////////////////////////////// + // make sure put-all works as expected // + ///////////////////////////////////////// + multiLingualWordToNumber.putAll(Map.of("One", BigDecimal.ONE, "Uno", BigDecimal.ONE, "EINS", BigDecimal.ONE, "dos", BIG_DECIMAL_TWO, "zwei", BIG_DECIMAL_TWO, "tres", BIG_DECIMAL_THREE)); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("oNe")); + assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("dos")); + assertEquals(BIG_DECIMAL_THREE, multiLingualWordToNumber.get("drei")); + assertEquals(3, multiLingualWordToNumber.size()); + assertEquals(3, multiLingualWordToNumber.entrySet().size()); + assertEquals(3, multiLingualWordToNumber.keySet().size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java index 8425fbfb..6a32ae17 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java @@ -61,7 +61,6 @@ public abstract class AbstractAPIAction apiActionUtil.setBackendMetaData(this.backendMetaData); apiActionUtil.setActionInput(actionInput); - apiActionUtil.setSession(session); } } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 209285e0..c4b478ab 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -61,6 +60,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -77,6 +79,7 @@ import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorExcep import com.kingsrook.qqq.backend.module.api.model.AuthorizationType; import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog; import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData; +import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendVariantSetting; import com.kingsrook.qqq.backend.module.api.model.metadata.APITableBackendDetails; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.BooleanUtils; @@ -114,7 +117,6 @@ public class BaseAPIActionUtil { private final QLogger LOG = QLogger.getLogger(BaseAPIActionUtil.class); - protected QSession session; // todo not commit - delete!! protected APIBackendMetaData backendMetaData; protected AbstractTableActionInput actionInput; @@ -777,8 +779,8 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); - return (record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField())); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); + return (record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.API_KEY, APIBackendVariantSetting.API_KEY))); } return (backendMetaData.getApiKey()); @@ -786,6 +788,18 @@ public class BaseAPIActionUtil + /*************************************************************************** + ** todo - once deprecated variant methods are removed from QBackendMetaData, + ** then we can remove the LegacyBackendVariantSetting enum, and this param. + ***************************************************************************/ + private String getVariantSettingSourceFieldName(APIBackendMetaData backendMetaData, LegacyBackendVariantSetting legacyBackendVariantSetting, APIBackendVariantSetting apiBackendVariantSetting) + { + Map map = CollectionUtils.nonNullMap(backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap()); + return map.getOrDefault(legacyBackendVariantSetting, map.get(apiBackendVariantSetting)); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -793,8 +807,11 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); - return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField()))); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); + return (Pair.of( + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.USERNAME, APIBackendVariantSetting.USERNAME)), + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.PASSWORD, APIBackendVariantSetting.PASSWORD)) + )); } return (Pair.of(backendMetaData.getUsername(), backendMetaData.getPassword())); @@ -802,50 +819,10 @@ public class BaseAPIActionUtil - /******************************************************************************* - ** For backends that use variants, look up the variant record (in theory, based - ** on an id in the session's backend variants map, then fetched from the backend's - ** variant options table. - *******************************************************************************/ - protected QRecord getVariantRecord() throws QException - { - Serializable variantId = getVariantId(); - GetInput getInput = new GetInput(); - getInput.setShouldMaskPasswords(false); - getInput.setTableName(backendMetaData.getVariantOptionsTableName()); - getInput.setPrimaryKey(variantId); - GetOutput getOutput = new GetAction().execute(getInput); - - QRecord record = getOutput.getRecord(); - if(record == null) - { - throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'")); - } - return record; - } - - - - /******************************************************************************* - ** Get the variant id from the session for the backend. - *******************************************************************************/ - protected Serializable getVariantId() throws QException - { - QSession session = QContext.getQSession(); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) - { - throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); - } - Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue()); - return variantId; - } - - - /******************************************************************************* ** *******************************************************************************/ - public String getOAuth2Token() throws OAuthCredentialsException, QException + String getOAuth2AccessTokenKey() throws QException { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // define the key that will be used in the backend's customValues map, to stash the access token. // @@ -854,9 +831,20 @@ public class BaseAPIActionUtil String accessTokenKey = "accessToken"; if(backendMetaData.getUsesVariants()) { - Serializable variantId = getVariantId(); + Serializable variantId = BackendVariantsUtil.getVariantId(backendMetaData); accessTokenKey = accessTokenKey + ":" + variantId; } + return (accessTokenKey); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public String getOAuth2Token() throws QException + { + String accessTokenKey = getOAuth2AccessTokenKey(); //////////////////////////////////////////////////////////////////////////////////////////////////////////////// // check for the access token in the backend meta data. if it's not there, then issue a request for a token. // @@ -944,8 +932,11 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); - return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableClientIdField()), record.getValueString(backendMetaData.getVariantOptionsTableClientSecretField()))); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); + return (Pair.of( + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_ID, APIBackendVariantSetting.CLIENT_ID)), + record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_SECRET, APIBackendVariantSetting.CLIENT_SECRET)) + )); } return (Pair.of(backendMetaData.getClientId(), backendMetaData.getClientSecret())); @@ -1245,7 +1236,8 @@ public class BaseAPIActionUtil if(!caughtAnOAuthExpiredToken) { LOG.info("OAuth Expired token for [" + table.getName() + "] - retrying"); - backendMetaData.withCustomValue("accessToken", null); + String accessTokenKey = getOAuth2AccessTokenKey(); + backendMetaData.withCustomValue(accessTokenKey, null); caughtAnOAuthExpiredToken = true; } else @@ -1480,9 +1472,9 @@ public class BaseAPIActionUtil ** Setter for session ** *******************************************************************************/ + @Deprecated(since = "wasn't used.") public void setSession(QSession session) { - this.session = session; } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java new file mode 100644 index 00000000..681977d3 --- /dev/null +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.api.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; + + +/******************************************************************************* + ** settings that the API backend module can get from a backend variant. + *******************************************************************************/ +public enum APIBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + API_KEY, + CLIENT_ID, + CLIENT_SECRET +} diff --git a/qqq-backend-module-filesystem/pom.xml b/qqq-backend-module-filesystem/pom.xml index 32cc274f..c389a018 100644 --- a/qqq-backend-module-filesystem/pom.xml +++ b/qqq-backend-module-filesystem/pom.xml @@ -50,6 +50,17 @@ aws-java-sdk-s3 1.12.261 + + org.apache.sshd + sshd-sftp + 2.14.0 + + + org.apache.sshd + sshd-sftp + 2.14.0 + + cloud.localstack localstack-utils @@ -57,6 +68,20 @@ test + + org.testcontainers + testcontainers + 1.15.3 + test + + + + net.java.dev.jna + jna + 5.7.0 + test + + org.apache.maven.plugins diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index fdf343c6..db3d3d05 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -25,9 +25,13 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; @@ -36,8 +40,12 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -47,12 +55,20 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -68,6 +84,8 @@ public abstract class AbstractBaseFilesystemAction { private static final QLogger LOG = QLogger.getLogger(AbstractBaseFilesystemAction.class); + protected QRecord backendVariantRecord = null; + /******************************************************************************* @@ -80,10 +98,26 @@ public abstract class AbstractBaseFilesystemAction + /*************************************************************************** + ** get the size of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Long getFileSize(FILE file); + + /*************************************************************************** + ** get the createDate of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Instant getFileCreateDate(FILE file); + + /*************************************************************************** + ** get the createDate of the specified file, null if not supported/available + ***************************************************************************/ + public abstract Instant getFileModifyDate(FILE file); + /******************************************************************************* - ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses. + ** List the files for a table - or optionally, just a single file name - + ** to be implemented in module-specific subclasses. *******************************************************************************/ - public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException; + public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedSingleFileName) throws QException; /******************************************************************************* ** Read the contents of a file - to be implemented in module-specific subclasses. @@ -107,7 +141,7 @@ public abstract class AbstractBaseFilesystemAction ** ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit *******************************************************************************/ - public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; + public abstract void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException; /******************************************************************************* ** Move a file from a source path, to a destination path. @@ -116,13 +150,21 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException; + + /******************************************************************************* ** e.g., with a base path of /foo/ ** and a table path of /bar/ ** and a file at /foo/bar/baz.txt ** give us just the baz.txt part. *******************************************************************************/ - public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable); + public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) + { + String tablePath = getFullBasePath(table, backend); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); + String withoutLeadingSlash = stripLeadingSlash(strippedPath); // todo - dangerous, do all backends really want this?? + return (withoutLeadingSlash); + } @@ -133,7 +175,17 @@ public abstract class AbstractBaseFilesystemAction public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase) { AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase); - String fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : ""; + + String basePath = metaData.getBasePath(); + if(backendBase.getUsesVariants()) + { + Map fieldNameMap = backendBase.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.BASE_PATH)) + { + basePath = backendVariantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.BASE_PATH)); + } + } + String fullPath = StringUtils.hasContent(basePath) ? basePath : ""; AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); if(StringUtils.hasContent(tableDetails.getBasePath())) @@ -164,6 +216,34 @@ public abstract class AbstractBaseFilesystemAction + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripLeadingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("^/+", "")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripTrailingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("/+$", "")); + } + + + /******************************************************************************* ** Get the backend metaData, type-checked as the requested type. *******************************************************************************/ @@ -202,112 +282,228 @@ public abstract class AbstractBaseFilesystemAction try { - QueryOutput queryOutput = new QueryOutput(queryInput); - QTableMetaData table = queryInput.getTable(); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); - int recordCount = 0; + QueryOutput queryOutput = new QueryOutput(queryInput); - FILE_LOOP: - for(FILE file : files) + String requestedPath = null; + List files = listFiles(table, queryInput.getBackend(), requestedPath); + + switch(tableDetails.getCardinality()) { - InputStream inputStream = readFile(file); - switch(tableDetails.getCardinality()) - { - case MANY: - { - LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); - switch(tableDetails.getRecordFormat()) - { - case CSV: - { - String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - fileContents = customizeFileContentsAfterReading(table, fileContents); - - if(queryInput.getRecordPipe() != null) - { - new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> - { - //////////////////////////////////////////////////////////////////////////////////////////// - // Before the records go into the pipe, make sure their backend details are added to them // - //////////////////////////////////////////////////////////////////////////////////////////// - addBackendDetailsToRecord(record, file); - })); - } - else - { - List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - queryOutput.addRecords(recordsInFile); - } - break; - } - case JSON: - { - String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - fileContents = customizeFileContentsAfterReading(table, fileContents); - - // todo - pipe support!! - List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - - queryOutput.addRecords(recordsInFile); - break; - } - default: - { - throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); - } - } - break; - } - case ONE: - { - //////////////////////////////////////////////////////////////////////////////// - // for one-record tables, put the entire file's contents into a single record // - //////////////////////////////////////////////////////////////////////////////// - String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); - byte[] bytes = inputStream.readAllBytes(); - - QRecord record = new QRecord() - .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase) - .withValue(tableDetails.getContentsFieldName(), bytes); - queryOutput.addRecord(record); - - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - // keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) // - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - recordCount++; - - //////////////////////////////////////////////////////////////////////////// - // break out of the file loop if we have hit the limit (if one was given) // - //////////////////////////////////////////////////////////////////////////// - if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null) - { - if(recordCount >= queryInput.getFilter().getLimit()) - { - break FILE_LOOP; - } - } - - break; - } - default: - { - throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); - } - } + case MANY -> completeExecuteQueryForManyTable(queryInput, queryOutput, files, table, tableDetails); + case ONE -> completeExecuteQueryForOneTable(queryInput, queryOutput, files, table, tableDetails); + default -> throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality()); } - return queryOutput; + return (queryOutput); } catch(Exception e) { LOG.warn("Error executing query", e); throw new QException("Error executing query", e); } + finally + { + postAction(); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void setRecordValueIfFieldNameHasContent(QRecord record, String fieldName, UnsafeSupplier valueSupplier) + { + if(StringUtils.hasContent(fieldName)) + { + try + { + record.setValue(fieldName, valueSupplier.get()); + } + catch(Exception e) + { + LOG.warn("Error setting record value for field", e, logPair("fieldName", fieldName)); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void completeExecuteQueryForOneTable(QueryInput queryInput, QueryOutput queryOutput, List files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException + { + int recordCount = 0; + List records = new ArrayList<>(); + + for(FILE file : files) + { + //////////////////////////////////////////////////////////////////////////////// + // for one-record tables, put the entire file's contents into a single record // + //////////////////////////////////////////////////////////////////////////////// + String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); + QRecord record = new QRecord(); + + setRecordValueIfFieldNameHasContent(record, tableDetails.getFileNameFieldName(), () -> filePathWithoutBase); + setRecordValueIfFieldNameHasContent(record, tableDetails.getBaseNameFieldName(), () -> stripAllPaths(filePathWithoutBase)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getSizeFieldName(), () -> getFileSize(file)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getCreateDateFieldName(), () -> getFileCreateDate(file)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getModifyDateFieldName(), () -> getFileModifyDate(file)); + + if(shouldHeavyFileContentsBeRead(queryInput, table, tableDetails)) + { + try(InputStream inputStream = readFile(file)) + { + byte[] bytes = inputStream.readAllBytes(); + record.withValue(tableDetails.getContentsFieldName(), bytes); + } + catch(Exception e) + { + record.addError(new SystemErrorStatusMessage("Error reading file contents: " + e.getMessage())); + } + } + else + { + Long size = record.getValueLong(tableDetails.getSizeFieldName()); + if(size != null) + { + if(record.getBackendDetails() == null) + { + record.setBackendDetails(new HashMap<>()); + } + + if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) + { + record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); + } + + ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size); + } + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // the listFiles method may have used a "path" criteria. // + // if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter filterForRecords = queryInput.getFilter(); + // if(filterForRecords != null) + // { + // filterForRecords = filterForRecords.clone(); + // CollectionUtils.nonNullList(filterForRecords.getCriteria()) + // .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria); + // } + + if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record)) + { + records.add(record); + } + } + + BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), records); + records = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), records); + queryOutput.addRecords(records); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private Serializable stripAllPaths(String filePath) + { + if(filePath == null) + { + return null; + } + + return (filePath.replaceFirst(".*/", "")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static boolean isPathEqualsCriteria(QFilterCriteria criteria) + { + return "path".equals(criteria.getFieldName()) && QCriteriaOperator.EQUALS.equals(criteria.getOperator()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void completeExecuteQueryForManyTable(QueryInput queryInput, QueryOutput queryOutput, List files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException, IOException + { + int recordCount = 0; + + for(FILE file : files) + { + try(InputStream inputStream = readFile(file)) + { + LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file))); + switch(tableDetails.getRecordFormat()) + { + case CSV -> + { + String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + if(queryInput.getRecordPipe() != null) + { + new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> + { + //////////////////////////////////////////////////////////////////////////////////////////// + // Before the records go into the pipe, make sure their backend details are added to them // + //////////////////////////////////////////////////////////////////////////////////////////// + addBackendDetailsToRecord(record, file); + })); + } + else + { + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + queryOutput.addRecords(recordsInFile); + } + } + case JSON -> + { + String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + fileContents = customizeFileContentsAfterReading(table, fileContents); + + // todo - pipe support!! + List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + + queryOutput.addRecords(recordsInFile); + } + default -> throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat()); + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static boolean shouldHeavyFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) + { + boolean doReadContents = true; + if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy()) + { + if(!queryInput.getShouldFetchHeavyFields()) + { + doReadContents = false; + } + } + return doReadContents; } @@ -319,7 +515,16 @@ public abstract class AbstractBaseFilesystemAction { QueryInput queryInput = new QueryInput(); queryInput.setTableName(countInput.getTableName()); - queryInput.setFilter(countInput.getFilter()); + + QQueryFilter filter = countInput.getFilter(); + if(filter != null) + { + filter = filter.clone(); + filter.setSkip(null); + filter.setLimit(null); + } + + queryInput.setFilter(filter); QueryOutput queryOutput = executeQuery(queryInput); CountOutput countOutput = new CountOutput(); @@ -353,11 +558,24 @@ public abstract class AbstractBaseFilesystemAction ** Method that subclasses can override to add pre-action things (e.g., setting up ** s3 client). *******************************************************************************/ - public void preAction(QBackendMetaData backendMetaData) + public void preAction(QBackendMetaData backendMetaData) throws QException { - ///////////////////////////////////////////////////////////////////// - // noop in base class - subclasses can add functionality if needed // - ///////////////////////////////////////////////////////////////////// + if(backendMetaData.getUsesVariants()) + { + this.backendVariantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData); + } + } + + + + /*************************************************************************** + ** Method that subclasses can override to add post-action things (e.g., closing resources) + ***************************************************************************/ + public void postAction() + { + ////////////////// + // noop in base // + ////////////////// } @@ -411,10 +629,18 @@ public abstract class AbstractBaseFilesystemAction { for(QRecord record : insertInput.getRecords()) { - String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); - writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); - record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); - output.addRecord(record); + try + { + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName())); + writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName())); + record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath); + output.addRecord(record); + } + catch(Exception e) + { + record.addError(new SystemErrorStatusMessage("Error writing file: " + e.getMessage())); + output.addRecord(record); + } } } else @@ -428,5 +654,63 @@ public abstract class AbstractBaseFilesystemAction { throw new QException("Error executing insert: " + e.getMessage(), e); } + finally + { + postAction(); + } } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected DeleteOutput executeDelete(DeleteInput deleteInput) throws QException + { + try + { + preAction(deleteInput.getBackend()); + + DeleteOutput output = new DeleteOutput(); + output.setRecordsWithErrors(new ArrayList<>()); + + QTableMetaData table = deleteInput.getTable(); + QBackendMetaData backend = deleteInput.getBackend(); + + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + if(tableDetails.getCardinality().equals(Cardinality.ONE)) + { + int deletedCount = 0; + for(Serializable primaryKey : deleteInput.getPrimaryKeys()) + { + try + { + deleteFile(table, stripDuplicatedSlashes(getFullBasePath(table, backend) + "/" + primaryKey)); + deletedCount++; + } + catch(Exception e) + { + String message = ObjectUtils.tryElse(() -> ExceptionUtils.getRootException(e).getMessage(), "Message not available"); + output.addRecordWithError(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withError(new SystemErrorStatusMessage("Error deleting file: " + message))); + } + } + output.setDeletedRecordCount(deletedCount); + } + else + { + throw (new NotImplementedException("Delete is not implemented for filesystem tables with cardinality: " + tableDetails.getCardinality())); + } + + return (output); + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + finally + { + postAction(); + } + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index f50f1b4c..307849b6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -41,6 +41,10 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private String contentsFieldName; private String fileNameFieldName; + private String baseNameFieldName; + private String sizeFieldName; + private String createDateFieldName; + private String modifyDateFieldName; @@ -281,4 +285,128 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails } } + + /******************************************************************************* + ** Getter for sizeFieldName + *******************************************************************************/ + public String getSizeFieldName() + { + return (this.sizeFieldName); + } + + + + /******************************************************************************* + ** Setter for sizeFieldName + *******************************************************************************/ + public void setSizeFieldName(String sizeFieldName) + { + this.sizeFieldName = sizeFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for sizeFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withSizeFieldName(String sizeFieldName) + { + this.sizeFieldName = sizeFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDateFieldName + *******************************************************************************/ + public String getCreateDateFieldName() + { + return (this.createDateFieldName); + } + + + + /******************************************************************************* + ** Setter for createDateFieldName + *******************************************************************************/ + public void setCreateDateFieldName(String createDateFieldName) + { + this.createDateFieldName = createDateFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for createDateFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withCreateDateFieldName(String createDateFieldName) + { + this.createDateFieldName = createDateFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDateFieldName + *******************************************************************************/ + public String getModifyDateFieldName() + { + return (this.modifyDateFieldName); + } + + + + /******************************************************************************* + ** Setter for modifyDateFieldName + *******************************************************************************/ + public void setModifyDateFieldName(String modifyDateFieldName) + { + this.modifyDateFieldName = modifyDateFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDateFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withModifyDateFieldName(String modifyDateFieldName) + { + this.modifyDateFieldName = modifyDateFieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for baseNameFieldName + *******************************************************************************/ + public String getBaseNameFieldName() + { + return (this.baseNameFieldName); + } + + + + /******************************************************************************* + ** Setter for baseNameFieldName + *******************************************************************************/ + public void setBaseNameFieldName(String baseNameFieldName) + { + this.baseNameFieldName = baseNameFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for baseNameFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withBaseNameFieldName(String baseNameFieldName) + { + this.baseNameFieldName = baseNameFieldName; + return (this); + } + + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index 19c1601e..7e61300f 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -22,14 +22,23 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; +import java.util.ArrayList; +import java.util.List; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; /******************************************************************************* @@ -53,6 +62,8 @@ public class FilesystemTableMetaDataBuilder private String basePath; private String glob; + private String contentsAdornmentFileNameField = "baseName"; + /******************************************************************************* @@ -60,26 +71,64 @@ public class FilesystemTableMetaDataBuilder *******************************************************************************/ public QTableMetaData buildStandardCardinalityOneTable() { + boolean includeCreateDate = true; AbstractFilesystemTableBackendDetails tableBackendDetails = switch(backend.getBackendType()) { - case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails(); + case S3BackendModule.BACKEND_TYPE -> + { + includeCreateDate = false; + yield new S3TableBackendDetails(); + } case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails(); + case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails(); default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType()); }; + List fields = new ArrayList<>(); + + fields.add((new QFieldMetaData("fileName", QFieldType.STRING))); + fields.add((new QFieldMetaData("baseName", QFieldType.STRING))); + fields.add((new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS))); + fields.add((new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))); + fields.add((new QFieldMetaData("contents", QFieldType.BLOB) + .withIsHeavy(true) + .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, contentsAdornmentFileNameField + )))); + + QFieldSection t3Section = SectionFactory.defaultT3("modifyDate"); + + AbstractFilesystemTableBackendDetails backendDetails = tableBackendDetails + .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withBaseNameFieldName("baseName") + .withContentsFieldName("contents") + .withSizeFieldName("size") + .withModifyDateFieldName("modifyDate") + .withBasePath(basePath) + .withGlob(glob); + + if(includeCreateDate) + { + fields.add((new QFieldMetaData("createDate", QFieldType.DATE_TIME))); + backendDetails.setCreateDateFieldName("createDate"); + + ArrayList t3FieldNames = new ArrayList<>(t3Section.getFieldNames()); + t3FieldNames.add(0, "createDate"); + t3Section.setFieldNames(t3FieldNames); + } + return new QTableMetaData() .withName(name) .withIsHidden(true) .withBackendName(backend.getName()) .withPrimaryKeyField("fileName") - .withField(new QFieldMetaData("fileName", QFieldType.INTEGER)) - .withField(new QFieldMetaData("contents", QFieldType.STRING)) - .withBackendDetails(tableBackendDetails - .withCardinality(Cardinality.ONE) - .withFileNameFieldName("fileName") - .withContentsFieldName("contents") - .withBasePath(basePath) - .withGlob(glob)); + .withFields(fields) + .withSection(SectionFactory.defaultT1("fileName")) + .withSection(SectionFactory.defaultT2("baseName", "contents", "size")) + .withSection(t3Section) + .withBackendDetails(backendDetails); } @@ -206,4 +255,35 @@ public class FilesystemTableMetaDataBuilder return (this); } + + /******************************************************************************* + ** Getter for contentsAdornmentFileNameField + *******************************************************************************/ + public String getContentsAdornmentFileNameField() + { + return (this.contentsAdornmentFileNameField); + } + + + + /******************************************************************************* + ** Setter for contentsAdornmentFileNameField + *******************************************************************************/ + public void setContentsAdornmentFileNameField(String contentsAdornmentFileNameField) + { + this.contentsAdornmentFileNameField = contentsAdornmentFileNameField; + } + + + + /******************************************************************************* + ** Fluent setter for contentsAdornmentFileNameField + *******************************************************************************/ + public FilesystemTableMetaDataBuilder withContentsAdornmentFileNameField(String contentsAdornmentFileNameField) + { + this.contentsAdornmentFileNameField = contentsAdornmentFileNameField; + return (this); + } + + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java index 026386b0..295aee06 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java @@ -130,7 +130,11 @@ public class SharedFilesystemBackendModuleUtils } else { - throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + /////////////////////////////////////////////////////////////////////////////////////////////// + // this happens in base class now, like, for query action, so, we think okay to just ignore. // + /////////////////////////////////////////////////////////////////////////////////////////////// + // throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + return (true); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index c9fb807f..cb7d9255 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -35,10 +35,14 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -61,11 +65,55 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Long getFileSize(File file) + { + return (file.length()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(File file) + { + try + { + Path path = file.toPath(); + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); + FileTime creationTime = attrs.creationTime(); + return creationTime.toInstant(); + } + catch(IOException e) + { + LOG.warn("Error getting file createDate", e, logPair("file", file)); + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(File file) + { + return Instant.ofEpochMilli(file.lastModified()); + } + + + /******************************************************************************* ** List the files for this table. *******************************************************************************/ @Override - public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { try { @@ -84,7 +132,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction for(String matchedFile : matchedFiles) { - if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails)) + boolean isMatch = true; + if(StringUtils.hasContent(requestedPath)) + { + QQueryFilter filter = new QQueryFilter(new QFilterCriteria(tableBackendDetails.getFileNameFieldName(), QCriteriaOperator.EQUALS, requestedPath)); + isMatch = SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails); + } + + if(isMatch) { rs.add(new File(fullPath + File.separatorChar + matchedFile)); } @@ -175,7 +230,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit *******************************************************************************/ @Override - public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + public void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException { File file = new File(fileReference); if(!file.exists()) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java index 6d304f00..6dda158d 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java @@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; -import org.apache.commons.lang.NotImplementedException; /******************************************************************************* ** *******************************************************************************/ -public class FilesystemDeleteAction implements DeleteInterface +public class FilesystemDeleteAction extends AbstractFilesystemAction implements DeleteInterface { /******************************************************************************* @@ -40,21 +39,19 @@ public class FilesystemDeleteAction implements DeleteInterface *******************************************************************************/ public DeleteOutput execute(DeleteInput deleteInput) throws QException { - throw new NotImplementedException("Filesystem delete not implemented"); - /* - try - { - DeleteResult rs = new DeleteResult(); - QTableMetaData table = deleteRequest.getTable(); + return (executeDelete(deleteInput)); + } - // return rs; - } - catch(Exception e) - { - throw new QException("Error executing delete: " + e.getMessage(), e); - } - */ + + /******************************************************************************* + ** Specify whether this particular module's update action can & should fetch + ** records before updating them, e.g., for audits or "not-found-checks" + *******************************************************************************/ + @Override + public boolean supportsPreFetchQuery() + { + return (false); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java index 68a319ca..604d4c2b 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java @@ -94,7 +94,7 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep if(VALUE_DELETE.equals(moveOrDelete)) { LOG.info("Deleting ETL source file: " + sourceFile); - actionBase.deleteFile(QContext.getQInstance(), table, sourceFile); + actionBase.deleteFile(table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java index 43867e99..bc0c3ec2 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java @@ -63,7 +63,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; -import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -314,13 +313,13 @@ public class FilesystemImporterStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException + private static void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws QException { if(removeFileAfterImport) { String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend); LOG.info("Removing source file", logPair("path", fullBasePath + "/" + sourceFileName), logPair("sourceTable", sourceTable.getName())); - sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName); + sourceActionBase.deleteFile(sourceTable, fullBasePath + "/" + sourceFileName); } else { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index 8a9a6272..05610fe6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; @@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3CountAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; @@ -112,6 +114,17 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new S3CountAction(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 91383c4b..3b8bcb09 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -24,15 +24,16 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import java.io.IOException; import java.io.InputStream; +import java.time.Instant; import java.util.List; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -56,11 +57,44 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); @@ -141,7 +175,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, String requestedPath, AbstractFilesystemTableBackendDetails tableDetails) throws QException { ////////////////////////////////////////////////////////////////////////////////////////////////// // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // @@ -96,38 +92,20 @@ public class S3Utils prefix = prefix.substring(0, prefix.indexOf('*')); } - /////////////////////////////////////////////////////////////////////////////////////////////////////// - // for a file-per-record (ONE) table, we may need to apply the filter to listing. // - // but for MANY tables, the filtering would be done on the records after they came out of the files. // - /////////////////////////////////////////////////////////////////////////////////////////////////////// - boolean useQQueryFilter = false; - if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality())) + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // optimization, to avoid listing whole bucket, for use-case where less than a whole bucket is requested // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(requestedPath)) { - useQQueryFilter = true; - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if there's a filter for single file, make that file name the "prefix" that we send to s3, so we just get back that 1 file. // - // as this will be a common case. // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(filter != null && useQQueryFilter) - { - if(filter.getCriteria() != null && filter.getCriteria().size() == 1) + if(!prefix.isEmpty()) { - QFilterCriteria criteria = filter.getCriteria().get(0); - if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) - { - if(!prefix.isEmpty()) - { - /////////////////////////////////////////////////////// - // remember, a prefix starting with / finds nothing! // - /////////////////////////////////////////////////////// - prefix += "/"; - } - - prefix += criteria.getValues().get(0); - } + /////////////////////////////////////////////////////// + // remember, a prefix starting with / finds nothing! // + /////////////////////////////////////////////////////// + prefix += "/"; } + + prefix += requestedPath; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -203,27 +181,7 @@ public class S3Utils continue; } - /////////////////////////////////////////////////////////////////////////////////// - // if we're a file-per-record table, and we have a filter, compare the key to it // - /////////////////////////////////////////////////////////////////////////////////// - if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails)) - { - continue; - } - rs.add(objectSummary); - - ///////////////////////////////////////////////////////////////// - // if we have a limit, and we've hit it, break out of the loop // - ///////////////////////////////////////////////////////////////// - if(filter != null && useQQueryFilter && filter.getLimit() != null) - { - if(rs.size() >= filter.getLimit()) - { - break; - } - } - } } while(listObjectsV2Result.isTruncated()); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java new file mode 100644 index 00000000..17c447dd --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java @@ -0,0 +1,170 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.AbstractSFTPAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPCountAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPDeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPInsertAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPStorageAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPUpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with SFTP filesystems (as a client) + *******************************************************************************/ +public class SFTPBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface +{ + public static final String BACKEND_TYPE = "sftp"; + + static + { + QBackendModuleDispatcher.registerBackendModule(new SFTPBackendModule()); + } + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractSFTPAction()); + } + + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return (BACKEND_TYPE); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (SFTPBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (SFTPTableBackendDetails.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new SFTPQueryAction(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new SFTPCountAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new SFTPInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new SFTPUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new SFTPDeleteAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return new SFTPStorageAction(); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java new file mode 100644 index 00000000..8d5330ff --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -0,0 +1,449 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import org.apache.sshd.client.SshClient; +import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.common.config.keys.KeyUtils; +import org.apache.sshd.sftp.client.SftpClient; +import org.apache.sshd.sftp.client.SftpClientFactory; +import org.apache.sshd.sftp.common.SftpException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Base class for all SFTP filesystem actions + *******************************************************************************/ +public class AbstractSFTPAction extends AbstractBaseFilesystemAction +{ + private static final QLogger LOG = QLogger.getLogger(AbstractSFTPAction.class); + + + + /*************************************************************************** + ** singleton implementing Initialization-on-Demand Holder idiom + ** to help ensure only a single SshClient object exists in a server. + ***************************************************************************/ + private static class SshClientManager + { + + /*************************************************************************** + ** + ***************************************************************************/ + private static class Holder + { + private static final SshClient INSTANCE = SshClient.setUpDefaultClient(); + + static + { + INSTANCE.start(); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static SshClient getInstance() + { + return Holder.INSTANCE; + } + } + + //////////////////////////////////////////////////////////////// + // open clientSessionFirst, then sftpClient // + // and close them in reverse (sftpClient, then clientSession) // + //////////////////////////////////////////////////////////////// + private ClientSession clientSession; + private SftpClient sftpClient; + + + + /******************************************************************************* + ** Set up the sftp utils object to be used for this action. + *******************************************************************************/ + @Override + public void preAction(QBackendMetaData backendMetaData) throws QException + { + super.preAction(backendMetaData); + + if(sftpClient != null) + { + LOG.debug("sftpClient object is already set - not re-setting it."); + return; + } + + try + { + SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData); + + String username = sftpBackendMetaData.getUsername(); + String password = sftpBackendMetaData.getPassword(); + String hostName = sftpBackendMetaData.getHostName(); + Integer port = sftpBackendMetaData.getPort(); + byte[] privateKey = sftpBackendMetaData.getPrivateKey(); + + if(backendMetaData.getUsesVariants()) + { + QRecord variantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData); + LOG.debug("Getting SFTP connection credentials from variant record", + logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()), + logPair("id", variantRecord.getValue("id")), + logPair("name", variantRecord.getRecordLabel())); + Map fieldNameMap = backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.USERNAME)) + { + username = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.USERNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PASSWORD)) + { + password = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.PASSWORD)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PRIVATE_KEY)) + { + privateKey = variantRecord.getValueByteArray(fieldNameMap.get(SFTPBackendVariantSetting.PRIVATE_KEY)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.HOSTNAME)) + { + hostName = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.HOSTNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PORT)) + { + port = variantRecord.getValueInteger(fieldNameMap.get(SFTPBackendVariantSetting.PORT)); + } + } + + makeConnection(username, hostName, port, password, privateKey); + } + catch(Exception e) + { + throw (new QException("Error setting up SFTP connection", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void postAction() + { + Consumer closer = closable -> + { + if(closable != null) + { + try + { + closable.close(); + } + catch(Exception e) + { + LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); + } + } + }; + + closer.accept(sftpClient); + closer.accept(clientSession); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected SftpClient makeConnection(String username, String hostName, Integer port, String password, byte[] privateKeyBytes) throws Exception + { + this.clientSession = SshClientManager.getInstance().connect(username, hostName, port).verify().getSession(); + + ////////////////////////////////////////////////////////////////////// + // if we have private key bytes, use them to add publicKey identity // + ////////////////////////////////////////////////////////////////////// + if(privateKeyBytes != null && privateKeyBytes.length > 0) + { + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(privateKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + PrivateKey privateKey = keyFactory.generatePrivate(keySpec); + PublicKey publicKey = KeyUtils.recoverPublicKey(privateKey); + clientSession.addPublicKeyIdentity(new KeyPair(publicKey, privateKey)); + } + + ////////////////////////////////////////////////// + // if we have a password, add password identity // + ////////////////////////////////////////////////// + if(StringUtils.hasContent(password)) + { + clientSession.addPasswordIdentity(password); + } + + clientSession.auth().verify(); + + this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); + return (this.sftpClient); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Long getFileSize(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getSize(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getCreateTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getModifyTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException + { + String fullPath = null; + try + { + fullPath = getFullBasePath(table, backendBase); + if(StringUtils.hasContent(requestedPath)) + { + fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + requestedPath + File.separatorChar); + } + + List rs = new ArrayList<>(); + + ///////////////////////////////////////////////////////////////////////////////////// + // at least in some cases, listing / seems to be interpreted by the server as // + // a listing from the root of the system, not just the user's dir. so, converting // + // paths starting with / to instead be ./ is giving us better results. // + ///////////////////////////////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = "." + fullPath; + } + + for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) + { + if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename())) + { + continue; + } + + if(dirEntry.getAttributes().isDirectory()) + { + // todo - recursive?? + continue; + } + + rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry)); + } + + return (rs); + } + catch(Exception e) + { + SftpException sftpException = ExceptionUtils.findClassInRootChain(e, SftpException.class); + if(sftpException != null) + { + throw new QUserFacingException("SFTP Exception listing [" + Objects.requireNonNullElse(fullPath, "") + "]: " + sftpException.getMessage()); + } + + throw (new QException("Error listing files", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public InputStream readFile(SFTPDirEntryWithPath dirEntry) throws IOException + { + return (sftpClient.read(getFullPathForFile(dirEntry))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException + { + sftpClient.put(new ByteArrayInputStream(contents), path); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getFullPathForFile(SFTPDirEntryWithPath dirEntry) + { + return (dirEntry.path() + "/" + dirEntry.dirEntry().getFilename()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException + { + try + { + sftpClient.remove(fileReference); + } + catch(Exception e) + { + throw (new FilesystemException("Error deleting file from SFTP", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected SftpClient getSftpClient(QBackendMetaData backend) throws QException + { + if(sftpClient == null) + { + preAction(backend); + } + + return (sftpClient); + } + + + + /*************************************************************************** + ** take a string, which is the contents of a PEM file (like a private key) + ** - and if it has the -----BEGIN...----- and -----END...---- lines, strip + ** them away, and strip away any whitespace, and then base-64 decode it. + ***************************************************************************/ + public static byte[] pemStringToDecodedBytes(String pemString) + { + String base64 = pemString.replaceAll("-----BEGIN (.*?)-----", "") + .replaceAll("-----END (.*?)-----", "") + .replaceAll("\\s", ""); + return Base64.getDecoder().decode(base64); + } +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java new file mode 100644 index 00000000..f882662c --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountAction extends AbstractSFTPAction implements CountInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public CountOutput execute(CountInput countInput) throws QException + { + return (executeCount(countInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java new file mode 100644 index 00000000..b13a84b6 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java @@ -0,0 +1,57 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteAction extends AbstractSFTPAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteOutput execute(DeleteInput deleteInput) throws QException + { + return (executeDelete(deleteInput)); + } + + + + /******************************************************************************* + ** Specify whether this particular module's update action can & should fetch + ** records before updating them, e.g., for audits or "not-found-checks" + *******************************************************************************/ + @Override + public boolean supportsPreFetchQuery() + { + return (false); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java new file mode 100644 index 00000000..aa1c81dc --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertAction extends AbstractSFTPAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertOutput execute(InsertInput insertInput) throws QException + { + return (super.executeInsert(insertInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java new file mode 100644 index 00000000..5759c7ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPQueryAction extends AbstractSFTPAction implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryOutput execute(QueryInput queryInput) throws QException + { + return (super.executeQuery(queryInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java new file mode 100644 index 00000000..ddcb40f1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.utils.SFTPOutputStream; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** (mass, streamed) storage action for sftp module + *******************************************************************************/ +public class SFTPStorageAction extends AbstractSFTPAction implements QStorageInterface +{ + + /******************************************************************************* + ** create an output stream in the storage backend - that can be written to, + ** for the purpose of inserting or writing a file into storage. + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + + SFTPOutputStream sftpOutputStream = new SFTPOutputStream(sftpClient, getFullPath(storageInput)); + return (sftpOutputStream); + } + catch(Exception e) + { + throw (new QException("Exception creating sftp output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullPath(StorageInput storageInput) throws QException + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + + ///////////////////////////////////////////////////////////// + // s3 seems to do better w/o leading slashes, so, strip... // + ///////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = fullPath.substring(1); + } + + return fullPath; + } + + + + /******************************************************************************* + ** create an input stream in the storage backend - that can be read from, + ** for the purpose of getting or reading a file from storage. + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + InputStream inputStream = sftpClient.read(getFullPath(storageInput)); + + return (inputStream); + } + catch(Exception e) + { + throw (new QException("Exception getting sftp input stream for file.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDownloadURL(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + //S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + //preAction(backend); + // + //AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + //String fullPath = getFullPath(storageInput); + //return (amazonS3.getUrl(backend.getBucketName(), fullPath).toString()); + } + catch(Exception e) + { + throw (new QException("Exception getting the sftp download URL.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void makePublic(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + } + catch(Exception e) + { + throw (new QException("Exception making sftp file publicly available.", e)); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java new file mode 100644 index 00000000..d31fbcbe --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java @@ -0,0 +1,438 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** action for testing credentials for an SFTP backend connection + *******************************************************************************/ +public class SFTPTestConnectionAction extends AbstractSFTPAction +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPTestConnectionTestOutput testConnection(SFTPTestConnectionTestInput input) + { + try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword(), input.getPrivateKey())) + { + SFTPTestConnectionTestOutput output = new SFTPTestConnectionTestOutput().withIsConnectionSuccess(true); + + if(StringUtils.hasContent(input.basePath)) + { + try + { + Iterable dirEntries = sftpClient.readDir(input.basePath); + + ///////////////////////////////////////////////////////////////////////// + // it seems like only the .iterator call throws if bad directory here. // + ///////////////////////////////////////////////////////////////////////// + dirEntries.iterator(); + output.setIsListBasePathSuccess(true); + } + catch(Exception e) + { + output.setIsListBasePathSuccess(false); + output.setListBasePathErrorMessage(e.getMessage()); + } + } + + return output; + } + catch(Exception e) + { + return new SFTPTestConnectionTestOutput().withIsConnectionSuccess(false).withConnectionErrorMessage(e.getMessage()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class SFTPTestConnectionTestInput + { + private String username; + private String hostName; + private Integer port; + private String password; + private String basePath; + private byte[] privateKey; + + + + /******************************************************************************* + ** Getter for username + ** + *******************************************************************************/ + public String getUsername() + { + return username; + } + + + + /******************************************************************************* + ** Setter for username + ** + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + ** + *******************************************************************************/ + public String getHostName() + { + return hostName; + } + + + + /******************************************************************************* + ** Setter for hostName + ** + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + ** + *******************************************************************************/ + public Integer getPort() + { + return port; + } + + + + /******************************************************************************* + ** Setter for port + ** + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPort(Integer port) + { + this.port = port; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + ** + *******************************************************************************/ + public String getPassword() + { + return password; + } + + + + /******************************************************************************* + ** Setter for password + ** + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for basePath + ** + *******************************************************************************/ + public String getBasePath() + { + return basePath; + } + + + + /******************************************************************************* + ** Setter for basePath + ** + *******************************************************************************/ + public void setBasePath(String basePath) + { + this.basePath = basePath; + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withBasePath(String basePath) + { + this.basePath = basePath; + return (this); + } + + + + /******************************************************************************* + ** Getter for privateKey + ** + *******************************************************************************/ + public byte[] getPrivateKey() + { + return privateKey; + } + + + + /******************************************************************************* + ** Setter for privateKey + ** + *******************************************************************************/ + public void setPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + } + + + + /******************************************************************************* + ** Fluent setter for privateKey + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + return (this); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class SFTPTestConnectionTestOutput + { + private Boolean isConnectionSuccess; + private String connectionErrorMessage; + + private Boolean isListBasePathSuccess; + private String listBasePathErrorMessage; + + + + /******************************************************************************* + ** Getter for isSuccess + ** + *******************************************************************************/ + public Boolean getIsConnectionSuccess() + { + return isConnectionSuccess; + } + + + + /******************************************************************************* + ** Setter for isSuccess + ** + *******************************************************************************/ + public void setIsConnectionSuccess(Boolean isSuccess) + { + this.isConnectionSuccess = isSuccess; + } + + + + /******************************************************************************* + ** Fluent setter for isSuccess + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withIsConnectionSuccess(Boolean isSuccess) + { + this.isConnectionSuccess = isSuccess; + return (this); + } + + + + /******************************************************************************* + ** Getter for connectionErrorMessage + ** + *******************************************************************************/ + public String getConnectionErrorMessage() + { + return connectionErrorMessage; + } + + + + /******************************************************************************* + ** Setter for connectionErrorMessage + ** + *******************************************************************************/ + public void setConnectionErrorMessage(String connectionErrorMessage) + { + this.connectionErrorMessage = connectionErrorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for connectionErrorMessage + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withConnectionErrorMessage(String connectionErrorMessage) + { + this.connectionErrorMessage = connectionErrorMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for listBasePathErrorMessage + *******************************************************************************/ + public String getListBasePathErrorMessage() + { + return (this.listBasePathErrorMessage); + } + + + + /******************************************************************************* + ** Setter for listBasePathErrorMessage + *******************************************************************************/ + public void setListBasePathErrorMessage(String listBasePathErrorMessage) + { + this.listBasePathErrorMessage = listBasePathErrorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for listBasePathErrorMessage + *******************************************************************************/ + public SFTPTestConnectionTestOutput withListBasePathErrorMessage(String listBasePathErrorMessage) + { + this.listBasePathErrorMessage = listBasePathErrorMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for isListBasePathSuccess + ** + *******************************************************************************/ + public Boolean getIsListBasePathSuccess() + { + return isListBasePathSuccess; + } + + + + /******************************************************************************* + ** Setter for isListBasePathSuccess + ** + *******************************************************************************/ + public void setIsListBasePathSuccess(Boolean isListBasePathSuccess) + { + this.isListBasePathSuccess = isListBasePathSuccess; + } + + + + /******************************************************************************* + ** Fluent setter for isListBasePathSuccess + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withIsListBasePathSuccess(Boolean isListBasePathSuccess) + { + this.isListBasePathSuccess = isListBasePathSuccess; + return (this); + } + + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java new file mode 100644 index 00000000..93b31a69 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateOutput execute(UpdateInput updateInput) throws QException + { + throw new NotImplementedException("SFTP update not implemented"); + /* + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List records = new ArrayList<>(); + rs.setRecords(records); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java new file mode 100644 index 00000000..06cae267 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model; + + +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** + *******************************************************************************/ +public record SFTPDirEntryWithPath(String path, SftpClient.DirEntry dirEntry) +{ +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java new file mode 100644 index 00000000..28ed667f --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java @@ -0,0 +1,230 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP backend meta data. + *******************************************************************************/ +public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData +{ + private String username; + private String password; + private String hostName; + private byte[] privateKey; + private Integer port; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPBackendMetaData() + { + super(); + setBackendType(SFTPBackendModule.class); + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPBackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public SFTPBackendMetaData withName(String name) + { + setName(name); + return this; + } + + + + /******************************************************************************* + ** Getter for username + *******************************************************************************/ + public String getUsername() + { + return (this.username); + } + + + + /******************************************************************************* + ** Setter for username + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + *******************************************************************************/ + public SFTPBackendMetaData withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + *******************************************************************************/ + public String getPassword() + { + return (this.password); + } + + + + /******************************************************************************* + ** Setter for password + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + *******************************************************************************/ + public SFTPBackendMetaData withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + *******************************************************************************/ + public String getHostName() + { + return (this.hostName); + } + + + + /******************************************************************************* + ** Setter for hostName + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + *******************************************************************************/ + public SFTPBackendMetaData withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + *******************************************************************************/ + public Integer getPort() + { + return (this.port); + } + + + + /******************************************************************************* + ** Setter for port + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + *******************************************************************************/ + public SFTPBackendMetaData withPort(Integer port) + { + this.port = port; + return (this); + } + + + + /******************************************************************************* + ** Getter for privateKey + *******************************************************************************/ + public byte[] getPrivateKey() + { + return (this.privateKey); + } + + + + /******************************************************************************* + ** Setter for privateKey + *******************************************************************************/ + public void setPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + } + + + + /******************************************************************************* + ** Fluent setter for privateKey + *******************************************************************************/ + public SFTPBackendMetaData withPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + return (this); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java new file mode 100644 index 00000000..2a702fc8 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum SFTPBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + HOSTNAME, + PORT, + BASE_PATH, + PRIVATE_KEY +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java new file mode 100644 index 00000000..3a1605e0 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP specific Extension of QTableBackendDetails + *******************************************************************************/ +public class SFTPTableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPTableBackendDetails() + { + super(); + setBackendType(SFTPBackendModule.class); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java new file mode 100644 index 00000000..b882c1ab --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.utils; + + +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.sshd.sftp.client.SftpClient; +import org.apache.sshd.sftp.common.SftpException; +import org.jetbrains.annotations.NotNull; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPOutputStream extends PipedOutputStream +{ + private static final QLogger LOG = QLogger.getLogger(SFTPOutputStream.class); + + private final SftpClient sftpClient; + private final String path; + + private final PipedInputStream pipedInputStream; + private final Future putFuture; + + private AtomicBoolean started = new AtomicBoolean(false); + private AtomicReference putException = new AtomicReference<>(null); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPOutputStream(SftpClient sftpClient, String path) throws IOException + { + pipedInputStream = new PipedInputStream(this, 32 * 1024); + + this.sftpClient = sftpClient; + this.path = path; + + putFuture = Executors.newSingleThreadExecutor().submit(() -> + { + try + { + started.set(true); + LOG.debug("Starting sftp put", logPair("path", path)); + sftpClient.put(pipedInputStream, path); + } + catch(Exception e) + { + putException.set(e); + LOG.error("Error putting SFTP output stream", e); + + try + { + pipedInputStream.close(); + } + catch(IOException ex) + { + LOG.error("Secondary error closing pipedInputStream after sftp put error", e); + } + + throw new RuntimeException(e); + } + }); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void write(@NotNull byte[] b) throws IOException + { + try + { + super.write(b); + } + catch(IOException e) + { + if(putException.get() != null) + { + if(putException.get() instanceof SftpException sftpException) + { + throw new IOException("Error performing SFTP put for path [" + path + "]: " + sftpException.getMessage()); + } + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error writing to SFTP output stream", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws IOException + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // don't try to close anything until we know that the sftpClient.put call's thread // + // has tried to start (otherwise, race condition could cause us to close things too early) // + ///////////////////////////////////////////////////////////////////////////////////////////// + int sleepLoops = 0; + while(!started.get() && sleepLoops++ <= 30) + { + SleepUtils.sleep(1, TimeUnit.SECONDS); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // closing the pipedOutputStream (super) causes things to flush and complete the put // + /////////////////////////////////////////////////////////////////////////////////////// + super.close(); + + //////////////////////////////// + // wait for the put to finish // + //////////////////////////////// + putFuture.get(60 - sleepLoops, TimeUnit.SECONDS); + + /////////////////////////////////////////////////////////////////////////////// + // in case the put-future never did start, throw explicitly mentioning that. // + /////////////////////////////////////////////////////////////////////////////// + if(sleepLoops >= 30) + { + throw (new Exception("future to can sftpClient.put() was never started.")); + } + } + catch(ExecutionException ee) + { + throw new IOException("Error performing SFTP put", ee); + } + catch(Exception e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error closing SFTP output stream", e); + } + finally + { + try + { + sftpClient.close(); + } + catch(IOException e) + { + LOG.error("Error closing SFTP client", e); + } + } + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index acad9100..233cad0d 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.FilesystemTableMetaDataBuilder; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; @@ -52,6 +55,10 @@ import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fil import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; import org.apache.commons.io.FileUtils; @@ -60,20 +67,26 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; - public static final String BACKEND_NAME_S3 = "s3"; - public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; - public static final String BACKEND_NAME_MOCK = "mock"; - public static final String BACKEND_NAME_MEMORY = "memory"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; + public static final String BACKEND_NAME_SFTP = "sftp"; + public static final String BACKEND_NAME_SFTP_WITH_VARIANTS = "sftpWithVariants"; + public static final String BACKEND_NAME_MOCK = "mock"; + public static final String BACKEND_NAME_MEMORY = "memory"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String TABLE_NAME_PERSON_SFTP = "person-sftp"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; + public static final String TABLE_NAME_SFTP_FILE = "sftp-file"; + public static final String TABLE_NAME_SFTP_FILE_VARIANTS = "sftp-file-with-variants"; + public static final String TABLE_NAME_VARIANT_OPTIONS = "variant-options-table"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; @@ -148,6 +161,7 @@ public class TestUtils qInstance.addBackend(defineS3Backend()); qInstance.addBackend(defineS3BackendSansPrefix()); qInstance.addTable(defineS3CSVPersonTable()); + qInstance.addTable(defineSFTPCSVPersonTable()); qInstance.addTable(defineS3BlobTable()); qInstance.addTable(defineS3BlobSansPrefixTable()); qInstance.addBackend(defineMockBackend()); @@ -155,6 +169,15 @@ public class TestUtils qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); + QBackendMetaData sftpBackend = defineSFTPBackend(); + qInstance.addBackend(sftpBackend); + qInstance.addTable(defineSFTPFileTable(sftpBackend)); + + QBackendMetaData sftpBackendWithVariants = defineSFTPBackendWithVariants(); + qInstance.addBackend(sftpBackendWithVariants); + qInstance.addTable(defineSFTPFileTableWithVariants(sftpBackendWithVariants)); + qInstance.addTable(defineVariantOptionsTable()); + definePersonCsvImporter(qInstance); return (qInstance); @@ -162,6 +185,21 @@ public class TestUtils + /*************************************************************************** + ** + ***************************************************************************/ + private static QTableMetaData defineVariantOptionsTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_VARIANT_OPTIONS) + .withBackendName(defineMemoryBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("basePath", QFieldType.STRING)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -379,6 +417,25 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineSFTPCSVPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_SFTP) + .withLabel("Person SFTP Table") + .withBackendName(BACKEND_NAME_SFTP) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields()) + .withBackendDetails(new SFTPTableBackendDetails() + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) + ); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -463,4 +520,77 @@ public class TestUtils MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); return (mockAuthenticationModule.createSession(defineInstance(), null)); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTable(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackend() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER) + .withName(BACKEND_NAME_SFTP)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTableWithVariants(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE_VARIANTS) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackendWithVariants() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + + //////////////////////////////////// + // only get basePath from variant // + //////////////////////////////////// + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TABLE_NAME_VARIANT_OPTIONS) + .withVariantTypeKey(TABLE_NAME_VARIANT_OPTIONS) + .withBackendSettingSourceFieldNameMap(Map.of( + SFTPBackendVariantSetting.BASE_PATH, "basePath" + )) + ) + .withName(BACKEND_NAME_SFTP_WITH_VARIANTS)); + } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index 26cb44ea..be82480e 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -26,9 +26,6 @@ import java.io.File; import java.io.IOException; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -41,7 +38,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -97,55 +93,59 @@ public class FilesystemBackendModuleTest ///////////////////////////////////////// // filter for a file name that's found // ///////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "BLOB-2.txt"); assertEquals(1, files.size()); assertEquals("BLOB-2.txt", files.get(0).getName()); - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "BLOB-1.txt"); assertEquals(1, files.size()); assertEquals("BLOB-1.txt", files.get(0).getName()); - /////////////////////////////////// - // filter for 2 names that exist // - /////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); - assertEquals(2, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // /////////////////////////////////// + // // filter for 2 names that exist // + // /////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + // assertEquals(2, files.size()); ///////////////////////////////////////////// // filter for a file name that isn't found // ///////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "NOT-FOUND.txt"); assertEquals(0, files.size()); - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); - assertEquals(1, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + // assertEquals(1, files.size()); - //////////////////////////////////////////////////// - // 2 criteria, and'ed, and can't match, so find 0 // - //////////////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); - assertEquals(0, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // //////////////////////////////////////////////////// + // // 2 criteria, and'ed, and can't match, so find 0 // + // //////////////////////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + // assertEquals(0, files.size()); - ////////////////////////////////////////////////// - // 2 criteria, or'ed, and both match, so find 2 // - ////////////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) - .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); - assertEquals(2, files.size()); + // ////////////////////////////////////////////////// + // // 2 criteria, or'ed, and both match, so find 2 // + // ////////////////////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + // .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + // assertEquals(2, files.size()); - ////////////////////////////////////// - // ensure unsupported filters throw // - ////////////////////////////////////// - assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) - .rootCause() - .hasMessageContaining("Unable to query filesystem table by field"); - assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) - .rootCause() - .hasMessageContaining("Unable to query filename field using operator"); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note that we used to try unsupported filters here, expecting them to throw - but those are // + // more-or-less now implemented in the base class's query method, so, no longer expected to throw here. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// } @@ -165,7 +165,7 @@ public class FilesystemBackendModuleTest List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); + filesystemBackendModule.getActionBase().deleteFile(table, filesBeforeDelete.get(0).getAbsolutePath()); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(), @@ -191,7 +191,7 @@ public class FilesystemBackendModuleTest List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + filesystemBackendModule.getActionBase().deleteFile(table, PATH_THAT_WONT_EXIST); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(), diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java index e43a667e..ff113a24 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java @@ -22,11 +22,19 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; -import org.apache.commons.lang.NotImplementedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -34,14 +42,25 @@ import static org.junit.jupiter.api.Assertions.assertThrows; *******************************************************************************/ public class FilesystemDeleteActionTest extends FilesystemActionTest { - /******************************************************************************* ** *******************************************************************************/ @Test - public void test() throws QException + public void testSuccessfulDeleteMultiple() throws QException { - assertThrows(NotImplementedException.class, () -> new FilesystemDeleteAction().execute(new DeleteInput())); + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount(); + + String filename1 = "A.txt"; + String filename2 = "B.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount()); + + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withPrimaryKeys(List.of(filename1, filename2))); + assertEquals(2, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + assertEquals(initialCount, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount()); } } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index c18ce4bc..b3b187de 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -26,9 +26,6 @@ import java.util.List; import java.util.UUID; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -38,7 +35,6 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -77,53 +73,59 @@ public class S3BackendModuleTest extends BaseS3Test ///////////////////////////////////////// // filter for a file name that's found // ///////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + files = actionBase.listFiles(table, backend, "BLOB-2.txt"); assertEquals(1, files.size()); assertThat(files.get(0).getKey()).contains("BLOB-2.txt"); - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + files = actionBase.listFiles(table, backend, "BLOB-1.txt"); assertEquals(1, files.size()); assertThat(files.get(0).getKey()).contains("BLOB-1.txt"); - /////////////////////////////////// - // filter for 2 names that exist // - /////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); - assertEquals(2, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // /////////////////////////////////// + // // filter for 2 names that exist // + // /////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + // assertEquals(2, files.size()); ///////////////////////////////////////////// // filter for a file name that isn't found // ///////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + files = actionBase.listFiles(table, backend, "NOT-FOUND.txt"); assertEquals(0, files.size()); - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); - assertEquals(1, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + // assertEquals(1, files.size()); - //////////////////////////////////////////////////// - // 2 criteria, and'ed, and can't match, so find 0 // - //////////////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); - assertEquals(0, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // //////////////////////////////////////////////////// + // // 2 criteria, and'ed, and can't match, so find 0 // + // //////////////////////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + // assertEquals(0, files.size()); - ////////////////////////////////////////////////// - // 2 criteria, or'ed, and both match, so find 2 // - ////////////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) - .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); - assertEquals(2, files.size()); + // ////////////////////////////////////////////////// + // // 2 criteria, or'ed, and both match, so find 2 // + // ////////////////////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + // .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + // assertEquals(2, files.size()); - ////////////////////////////////////// - // ensure unsupported filters throw // - ////////////////////////////////////// - assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) - .hasMessageContaining("Unable to query filesystem table by field"); - assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) - .hasMessageContaining("Unable to query filename field using operator"); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note that we used to try unsupported filters here, expecting them to throw - but those are // + // more-or-less now implemented in the base class's query method, so, no longer expected to throw here. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// } @@ -145,7 +147,7 @@ public class S3BackendModuleTest extends BaseS3Test S3BackendModule s3BackendModule = new S3BackendModule(); AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); - actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); + actionBase.deleteFile(table, s3ObjectSummariesBeforeDelete.get(0).getKey()); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), @@ -174,7 +176,7 @@ public class S3BackendModuleTest extends BaseS3Test AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); String path = "//" + s3ObjectSummariesBeforeDelete.get(0).getKey().replaceAll("/", "//"); - actionBase.deleteFile(qInstance, table, "//" + path); + actionBase.deleteFile(table, "//" + path); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), @@ -201,7 +203,7 @@ public class S3BackendModuleTest extends BaseS3Test S3BackendModule s3BackendModule = new S3BackendModule(); AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); - actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + actionBase.deleteFile(table, PATH_THAT_WONT_EXIST); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(), diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java index 6b1ba2fa..ce01677f 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java @@ -22,12 +22,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; +import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; -import org.apache.commons.lang.NotImplementedException; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -42,7 +49,42 @@ public class S3DeleteActionTest extends BaseS3Test @Test public void test() throws QException { - assertThrows(NotImplementedException.class, () -> new S3DeleteAction().execute(new DeleteInput())); + QInstance qInstance = TestUtils.defineInstance(); + + int initialCount = count(TestUtils.TABLE_NAME_BLOB_S3); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob."))); + + S3InsertAction insertAction = new S3InsertAction(); + insertAction.setS3Utils(getS3Utils()); + insertAction.execute(insertInput); + + assertEquals(initialCount + 1, count(TestUtils.TABLE_NAME_BLOB_S3)); + + S3DeleteAction deleteAction = new S3DeleteAction(); + deleteAction.setS3Utils(getS3Utils()); + DeleteOutput deleteOutput = deleteAction.execute(new DeleteInput(TestUtils.TABLE_NAME_BLOB_S3).withPrimaryKeys(List.of("file2.txt"))); + assertEquals(1, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + + assertEquals(initialCount, count(TestUtils.TABLE_NAME_BLOB_S3)); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + private Integer count(String tableName) throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(tableName); + S3CountAction s3CountAction = new S3CountAction(); + s3CountAction.setS3Utils(getS3Utils()); + CountOutput countOutput = s3CountAction.execute(countInput); + return countOutput.getCount(); } } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java new file mode 100644 index 00000000..e7093402 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.module.filesystem.BaseTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.MountableFile; + + +/******************************************************************************* + ** Base class for tests that want to be able to work with sftp testcontainer + *******************************************************************************/ +public class BaseSFTPTest extends BaseTest +{ + public static final int PORT = 22; + public static final String USERNAME = "testuser"; + public static final String PASSWORD = "testpass"; + public static final String HOST_NAME = "localhost"; + + public static final String BACKEND_FOLDER = "upload"; + public static final String TABLE_FOLDER = "files"; + public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER; + + protected static GenericContainer sftpContainer; + private static Integer currentPort; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @BeforeAll + static void setUp() throws Exception + { + sftpContainer = new GenericContainer<>("atmoz/sftp:latest") + .withExposedPorts(PORT) + .withCommand(USERNAME + ":" + PASSWORD + ":1001"); + + sftpContainer.start(); + + for(int i = 0; i < 5; i++) + { + copyFileToContainer("files/testfile.txt", REMOTE_DIR + "/testfile-" + i + ".txt"); + } + + grantUploadFilesDirWritePermission(); + + /////////////////////////////////////////////////// + // add our test-only public key to the container // + /////////////////////////////////////////////////// + String sshDir = "/home/" + USERNAME + "/.ssh"; + sftpContainer.execInContainer("mkdir", sshDir); + sftpContainer.execInContainer("chmod", "700", sshDir); + sftpContainer.execInContainer("chown", USERNAME, sshDir); + copyFileToContainer("test-only-key.pub", sshDir + "/authorized_keys"); + sftpContainer.execInContainer("chmod", "600", sshDir + "/authorized_keys"); + sftpContainer.execInContainer("chown", USERNAME, sshDir + "/authorized_keys"); + + currentPort = sftpContainer.getMappedPort(22); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void copyFileToContainer(String sourceFileClasspathResourceName, String fullRemotePath) + { + sftpContainer.copyFileToContainer(MountableFile.forClasspathResource(sourceFileClasspathResourceName), fullRemotePath); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void rmrfInContainer(String fullRemotePath) throws Exception + { + sftpContainer.execInContainer("rm", "-rf", fullRemotePath); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @AfterAll + static void tearDown() + { + if(sftpContainer != null) + { + sftpContainer.stop(); + } + } + + + + /******************************************************************************* + ** Getter for currentPort + ** + *******************************************************************************/ + public static Integer getCurrentPort() + { + return currentPort; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void revokeUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("444"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void grantUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("777"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void setUploadFilesDirPermission(String mode) throws Exception + { + sftpContainer.execInContainer("chmod", mode, "/home/testuser/upload/files"); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception + { + sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path); + } +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java new file mode 100644 index 00000000..643fe2ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCount1() throws QException + { + CountInput countInput = initCountRequest(); + SFTPCountAction countAction = new SFTPCountAction(); + CountOutput countOutput = countAction.execute(countInput); + Assertions.assertEquals(5, countOutput.getCount(), "Expected # of rows from unfiltered count"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private CountInput initCountRequest() throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + return countInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java new file mode 100644 index 00000000..15d74d3b --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java @@ -0,0 +1,114 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteActionTest extends BaseSFTPTest +{ + private String filesBasename = "delete-test-"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() throws Exception + { + rmrfInContainer(REMOTE_DIR + "/" + filesBasename + "*"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSuccessfulDeleteMultiple() throws QException + { + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount(); + + String filename1 = filesBasename + "A.txt"; + String filename2 = filesBasename + "B.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SFTP_FILE).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_SFTP_FILE).withPrimaryKeys(List.of(filename1, filename2))); + assertEquals(2, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + assertEquals(initialCount, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFailedDelete() throws Exception + { + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount(); + + String filename1 = filesBasename + "C.txt"; + String filename2 = filesBasename + "D.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SFTP_FILE).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + + sftpContainer.execInContainer("chmod", "000", REMOTE_DIR); + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_SFTP_FILE).withPrimaryKeys(List.of(filename1, filename2))); + sftpContainer.execInContainer("chmod", "777", REMOTE_DIR); + + assertEquals(0, deleteOutput.getDeletedRecordCount()); + assertEquals(2, deleteOutput.getRecordsWithErrors().size()); + assertThat(deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString()).contains("Error deleting file: Permission denied"); + assertThat(deleteOutput.getRecordsWithErrors().get(1).getErrorsAsString()).contains("Error deleting file: Permission denied"); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java new file mode 100644 index 00000000..c648021c --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java @@ -0,0 +1,120 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.IOException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOne() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + assertThat(insertOutput.getRecords()) + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseSFTPTest.BACKEND_FOLDER)); + + QRecord record = insertOutput.getRecords().get(0); + String fullPath = record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH); + assertThat(record.getErrors()).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOnePermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + + QRecord record = insertOutput.getRecords().get(0); + assertThat(record.getErrors()).isNotEmpty(); + assertThat(record.getErrors().get(0).getMessage()).contains("Error writing file: Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityMany() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_SFTP); + insertInput.setRecords(List.of( + new QRecord().withValue("id", "1").withValue("firstName", "Bob") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + assertThatThrownBy(() -> insertAction.execute(insertInput)) + .hasRootCauseInstanceOf(NotImplementedException.class); + } +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java new file mode 100644 index 00000000..35f87644 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for SFTPQueryAction + *******************************************************************************/ +class SFTPQueryActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSimpleQuery() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSimpleQueryForOneFile() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "testfile-1.txt"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryVariantsTable() throws Exception + { + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_VARIANT_OPTIONS).withRecords(List.of( + new QRecord().withValue("id", 1).withValue("basePath", BaseSFTPTest.BACKEND_FOLDER), + new QRecord().withValue("id", 2).withValue("basePath", "empty-folder"), + new QRecord().withValue("id", 3).withValue("basePath", "non-existing-path") + ))); + + mkdirInSftpContainerUnderHomeTestuser("empty-folder/files"); + + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'variant-options-table' for Backend"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 2)); + queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(0, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 3)); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .rootCause() + .hasMessageContaining("No such file"); + + // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java new file mode 100644 index 00000000..ed49cc2d --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +public class SFTPStorageActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSmall() throws Exception + { + String data = "Hellooo, Storage."; + runTest(data); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testPermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + String data = "oops!"; + assertThatThrownBy(() -> runTest(data)) + .hasRootCauseInstanceOf(IOException.class) + .rootCause() + .hasMessageContaining("Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLarge() throws Exception + { + String data = StringUtils.join("!", Collections.nCopies(5_000_000, "Hello")); + runTest(data); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void runTest(String data) throws QException, IOException + { + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_SFTP_FILE).withReference("fromStorageAction.txt"); + + StorageAction storageAction = new StorageAction(); + OutputStream outputStream = storageAction.createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = storageAction.getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data.length(), byteArrayOutputStream.toString(StandardCharsets.UTF_8).length()); + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java new file mode 100644 index 00000000..d569c2a9 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java @@ -0,0 +1,206 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for SFTPTestConnectionAction + *******************************************************************************/ +class SFTPTestConnectionActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessWithoutPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessWithPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertTrue(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessfulConnectFailedPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME) + .withBasePath("no-such-path"); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertFalse(output.getIsListBasePathSuccess()); + assertNotNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadUsername() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername("not-" + BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadPassword() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword("not-" + BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadHostname() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName("not-" + BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadPort() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(10 * BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testConnectViaPublicKey() throws Exception + { + try(InputStream resourceAsStream = getClass().getResourceAsStream("/test-only-key")) + { + byte[] privateKeyBytes = AbstractSFTPAction.pemStringToDecodedBytes(StringUtils.join("", IOUtils.readLines(resourceAsStream, StandardCharsets.UTF_8))); + + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPrivateKey(privateKeyBytes) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java new file mode 100644 index 00000000..2f025320 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPUpdateAction().execute(new UpdateInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/README.md b/qqq-backend-module-filesystem/src/test/resources/README.md new file mode 100644 index 00000000..abaabf7a --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/README.md @@ -0,0 +1,11 @@ +The `test-only-key` / `test-only-key.pub` key pair in this directory was generated via: + +```shell +ssh-keygen -t rsa -b 4096 -m PEM -f test-only-key +openssl pkcs8 -topk8 -inform PEM -in test-only-key -outform PEM -nocrypt -out test-only-key-kpcs8.pem +cp test-only-key-kpcs8.pem .../src/test/resources/test-only-key +``` + +It is NOT meant to be used as a secure key in ANY environment. + +It is included in this repo ONLY to be used for basic unit testing. \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt new file mode 100644 index 00000000..eab448b1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt @@ -0,0 +1,3 @@ +This is a file. + +It is a test. diff --git a/qqq-backend-module-filesystem/src/test/resources/test-only-key b/qqq-backend-module-filesystem/src/test/resources/test-only-key new file mode 100644 index 00000000..c3ca2a4d --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/test-only-key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDGEYxHZMPM6A+4 +OtayotUrB94AqYow1PcK5HzY8psKuHYejurugwFBbrFHq+dOso6EkLKcSpjvEX+X +Se+m3CDIC6qVuLBS6fmWe6wo73fhlqyEP6rslz//v5+Di8fhCPSfa8y4X5RLbkDR +yrL/ilmIEHUcmCU82SFFp9C10W+Ir9SZbjAoiRHT+X/WWmXYnMrfLHl38xxdWr8O +qXEmLaVxmSaLnWAmMXRr4Nqrk7r7VVq899CZSJMVLSfqeiJPqyLkwOPcYjzOKft0 +hYAhIXCUsEK4Wv7R1t9SooC7nqE2mY/htBcrPPyDhTT4Kl0zAD2MNloNUUQsTnic +t3FCUy0pB9zr26bZ4YHBIvdReTWcx6IlH2U28V9ktxWRJe/ESqay4RTfif24aWya +WzD9+yTowTdPJbMaQZI5XGYJBfFtXr8aC8Di+VuILbriiuLk1iPuj1p2R3JXepON +sVRDBm10SmHdETcsZXB3glqqiV+mFGmHguPXSo1WZmP79+kj/phwtn2wFKRLwi6k +QNeqOxLbamRtj7y4M2sphEC7Cih1w4J+CzjZE8PC7+Bck+ldvmdtW6+wkkhLgli9 +SvDwe2pM8eTWTznzs1Qj9My5p1v0uvIo0xADtpITzb3kQtl9+2C7uqscjJLIpc0g +VM5q55bFgxUp32G+HKUT8TI98ZrbEwIDAQABAoICAQC1f2MCEO3zGDs/YHtYmimo +Er917+W3tY7jJljZHAbCniFvAxt4kAdYhCxjNrzwumIqS8W/vgPCHlDCu3eleVV4 +umgIZoL8l3akVJN/t2AtEbroPMdNoZN9sYRkMHRqW6B9bXTdBoiHTnKLS6kWzRoZ +uqr2Ft0YkwcQIyT3VwFTSXwRVI1At8nkal6gd5mYEqU8OC7eoaG+Ued9cftDNtTB +8csGaKGwneTG7fay/t56bM6HFrbJn11YLFbFYEGMq49/+tlSG5sIeCP5tFOjCFd3 +iMS61ndmpfVibZJ1Wnjz5WeZNUN91Za2lhvhxEA++dtsXmnKhktoJkgTo21fj4Ry +1DbdysR5W0vz+HQ5usT/fWAypKj4KSlP6rsL/zkqmlxW1qE/nwzK6UN6kVWvO1Sm +6EcZIQKTRpkGFXluOJL/Bkqg+4Ayl9G3veLON5yTOB367m1haI/oIgr+jOKrkzdG +PWWNZ78USCT9YD/P2vHkwVQ/uzF3kHhLL6n8hzunFpJk+5eScXsyXTUjDCn4HurE +A8Z1FHLOeu4EEKODEdg1PCa34/8Z0K+88G5G/sTDWLf2MjywdCSDzKxYiSpQW0gt +5I5WgQG8CiGHbysBWMnqjoG1J+QpAUDzT29CuLhVbn4i0uiTe0AZ0xLjravhd5dW +dlItzvOY1rj5jHPs5hiWkQKCAQEA97jAuCzPw2Ats2QUfpnqEd/ITKRYTtK6zo+c +sdc1JH0RnzxCeBr2BApSrfMbqAMrPaiUKk5FUqpF5q47AXGrYFlaAPe7cq+Lr97N +LRi7NHM9RdF1myollCghwgKkq8qUe64eNAypMtGOdjrMh7kC879udESYXLjqgT5H +wQbHF5IRvg5vBkVKxzLrg37lX1f0MVGdB8VxA+QRQ4egFuQApPvMLFVenYZL165r +u3OIpOQWcJ91L5VzN5jJMR1x8VWFR6iD4PxachdmD8qaNxfwWKTTWxkpwTXfW1EI +68NZ2s9RmuRbJEfOtiznfzdVL+lAibeMe7dvUMzIevr0hPzMNwKCAQEAzLADz691 +9bJniRVFUkHlbXRV7k0sRxAXNzGmar8xpnG3wJaW+zSsj8Nnr+aSBGgGWmmldEn7 +tiHlozrjWUgUEsRJKObiCrdGsTC7+0flQLs1bukLfgZSnP3oMDYAgq15Vaw49oCU +M4KxRGfkPEhwP/DZClHsYkPr6HegT2/21z8AFHTAxknGjWWGJJAFxwaog7Akugdy +gXLb7lU4SjCJdb5tR1c7aDEUOvDDu1iffhWtt5Tp9BL0dKlN4M+6XZvqSNiVlN6P +BB5gDuSa0qEewIbMWiT4rcvE7gCSXFEWPnGbtHU7QcI4Wx1F75Y4CRgs2rRlnj9j +bVAsRNIOTqkSBQKCAQEAliEL+xJ9X6TcTYnrucZBy09aLsizFCI2QJVcm5MXi+OY +WG7Gwc9lJZG0BeP98Nbqz9Vo5jLFZJH5BxK0g+2FtUCxgUCiA6FMAOwAYMJKQkFM +8xE8OytR1vZzbwb3EX4WetZNS7IYoMnLku+ToPWJSnvLzv77b8ZJqMY76knXQvut +cQeCVcSMyyia/vhavmupfHI/vsPz+C2yIMEDTpwjn9lSJdQfIUyQjkgQ1mvwdi4d +Q2gANzRVvW4FEJUNxvrTaVhBhIqrrdVsb0mUKKuDZ9WMmfsoCQZDNS5pP6kGvctD +Y6HdcqFqL5ILQlggcobkLBJnO1syRT+2iIGqyyYCBQKCAQAbwy/xJnJQbe8/F6R8 +YLW2n9Xb6Zm81cDgWpqg1eftFHWA6Kv3zJAvO6i/of1iHZ3m+3dWi4ZZkMVt21nk +zTLzzK3Dn3U/UNaEyABnN7wviHTZ40AMyty/sGyixWBSWScg6KgdPxla1zol9hVt +28Fl2swFa1EtjtrbgAY9YAlR7pibLa7L9ku49/E22lX+RbfrjKOem837ItITxHlL +DsRGNRrrVziWjDmbOPbDXWTcnCIgyVDmKv//JsuKV4KGmdQwJzg6pekt/NS4kGcz +dGkQYfgrreIQ6JeAVJGFdfYXaB9fXZs48xfju9e1hGF7Uk0bKOazjRN2Sy6F8xu/ +rYzlAoIBAEjY7u3Jmntn1AYsbuy9wTblKl1IaZP4ST+X0/dLtvW8ZLsx0jPGwMXx +xmOku5OGqPjCn5i8Ws2KS8O6O+7lGm/CHXvmDpozD3wpjnJ64SgoLnjrT8R78TEJ +UjsGQfR7ofSj4heR7TgEPp+n0SXse3qERd6VZ5YPuzGva1iVJogErwI58QU2QaxQ +0ONV6F8oZuXjUs9KRhXQ8W0i87m0P7/ZumhqPaQqY/MeAYF/ED5C6ETKISxaDqs/ +zd/jf6uPZL6P4DPWcw7cSk5/aNZZ0P+/BkEX33WHBDSdVyHC+ydMcYZBrrlWKoSt +sNTITZbKrQB4hwHdawpMHxh+5mRXLk0= +-----END PRIVATE KEY----- diff --git a/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub b/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub new file mode 100644 index 00000000..02a25a74 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGEYxHZMPM6A+4OtayotUrB94AqYow1PcK5HzY8psKuHYejurugwFBbrFHq+dOso6EkLKcSpjvEX+XSe+m3CDIC6qVuLBS6fmWe6wo73fhlqyEP6rslz//v5+Di8fhCPSfa8y4X5RLbkDRyrL/ilmIEHUcmCU82SFFp9C10W+Ir9SZbjAoiRHT+X/WWmXYnMrfLHl38xxdWr8OqXEmLaVxmSaLnWAmMXRr4Nqrk7r7VVq899CZSJMVLSfqeiJPqyLkwOPcYjzOKft0hYAhIXCUsEK4Wv7R1t9SooC7nqE2mY/htBcrPPyDhTT4Kl0zAD2MNloNUUQsTnict3FCUy0pB9zr26bZ4YHBIvdReTWcx6IlH2U28V9ktxWRJe/ESqay4RTfif24aWyaWzD9+yTowTdPJbMaQZI5XGYJBfFtXr8aC8Di+VuILbriiuLk1iPuj1p2R3JXepONsVRDBm10SmHdETcsZXB3glqqiV+mFGmHguPXSo1WZmP79+kj/phwtn2wFKRLwi6kQNeqOxLbamRtj7y4M2sphEC7Cih1w4J+CzjZE8PC7+Bck+ldvmdtW6+wkkhLgli9SvDwe2pM8eTWTznzs1Qj9My5p1v0uvIo0xADtpITzb3kQtl9+2C7uqscjJLIpc0gVM5q55bFgxUp32G+HKUT8TI98ZrbEw== test-only-key diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java index 5a84d73b..928720dc 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java @@ -56,6 +56,8 @@ public class RDBMSBackendModule implements QBackendModuleInterface { private static final QLogger LOG = QLogger.getLogger(RDBMSBackendModule.class); + public static final String NAME = "rdbms"; + static { QBackendModuleDispatcher.registerBackendModule(new RDBMSBackendModule()); @@ -66,7 +68,7 @@ public class RDBMSBackendModule implements QBackendModuleInterface *******************************************************************************/ public String getBackendType() { - return ("rdbms"); + return NAME; } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 68ecce54..e5511ed8 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -39,8 +39,6 @@ import java.util.ListIterator; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; -import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -63,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -80,9 +79,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSFieldMetaData; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -101,6 +101,9 @@ public abstract class AbstractRDBMSAction private static Memoization doesSelectClauseRequireDistinctMemoization = new Memoization() .withTimeout(Duration.ofDays(365)); + private RDBMSBackendMetaData backendMetaData; + private RDBMSActionStrategyInterface actionStrategy; + /******************************************************************************* @@ -313,9 +316,9 @@ public abstract class AbstractRDBMSAction } joinClauseList.add(escapeIdentifier(baseTableOrAlias) - + "." + escapeIdentifier(getColumnName(leftTable.getField(joinOn.getLeftField()))) - + " = " + escapeIdentifier(joinTableOrAlias) - + "." + escapeIdentifier(getColumnName((rightTable.getField(joinOn.getRightField()))))); + + "." + escapeIdentifier(getColumnName(leftTable.getField(joinOn.getLeftField()))) + + " = " + escapeIdentifier(joinTableOrAlias) + + "." + escapeIdentifier(getColumnName((rightTable.getField(joinOn.getRightField()))))); } if(CollectionUtils.nullSafeHasContents(queryJoin.getSecurityCriteria())) @@ -479,172 +482,25 @@ public abstract class AbstractRDBMSAction JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(criterion.getFieldName()); - List values = criterion.getValues() == null ? new ArrayList<>() : new ArrayList<>(criterion.getValues()); - QFieldMetaData field = fieldAndTableNameOrAlias.field(); - String column = escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(field)); - String clause = column; - Integer expectedNoOfParams = null; - switch(criterion.getOperator()) - { - case EQUALS -> - { - clause += " = ?"; - expectedNoOfParams = 1; - } - case NOT_EQUALS -> - { - clause += " != ?"; - expectedNoOfParams = 1; - } - case NOT_EQUALS_OR_IS_NULL -> - { - clause += " != ? OR " + column + " IS NULL "; - expectedNoOfParams = 1; - } - case IN -> - { - if(values.isEmpty()) - { - /////////////////////////////////////////////////////// - // if there are no values, then we want a false here // - /////////////////////////////////////////////////////// - clause = " 0 = 1 "; - } - else - { - clause += " IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; - } - } - case IS_NULL_OR_IN -> - { - clause += " IS NULL "; + List values = criterion.getValues() == null ? new ArrayList<>() : new ArrayList<>(criterion.getValues()); + QFieldMetaData field = fieldAndTableNameOrAlias.field(); + String column = escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(field)); + StringBuilder clause = new StringBuilder(); - if(!values.isEmpty()) - { - clause += " OR " + column + " IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; - } - } - case NOT_IN -> + RDBMSActionStrategyInterface actionStrategy = getActionStrategy(); + + RDBMSFieldMetaData rdbmsFieldMetaData = RDBMSFieldMetaData.of(field); + if(rdbmsFieldMetaData != null) + { + RDBMSActionStrategyInterface fieldActionStrategy = rdbmsFieldMetaData.getActionStrategy(); + if(fieldActionStrategy != null) { - if(values.isEmpty()) - { - ////////////////////////////////////////////////////// - // if there are no values, then we want a true here // - ////////////////////////////////////////////////////// - clause = " 1 = 1 "; - } - else - { - clause += " NOT IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; - } + actionStrategy = fieldActionStrategy; } - case LIKE -> - { - clause += " LIKE ?"; - expectedNoOfParams = 1; - } - case NOT_LIKE -> - { - clause += " NOT LIKE ?"; - expectedNoOfParams = 1; - } - case STARTS_WITH -> - { - clause += " LIKE ?"; - ActionHelper.editFirstValue(values, (s -> s + "%")); - expectedNoOfParams = 1; - } - case ENDS_WITH -> - { - clause += " LIKE ?"; - ActionHelper.editFirstValue(values, (s -> "%" + s)); - expectedNoOfParams = 1; - } - case CONTAINS -> - { - clause += " LIKE ?"; - ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); - expectedNoOfParams = 1; - } - case NOT_STARTS_WITH -> - { - clause += " NOT LIKE ?"; - ActionHelper.editFirstValue(values, (s -> s + "%")); - expectedNoOfParams = 1; - } - case NOT_ENDS_WITH -> - { - clause += " NOT LIKE ?"; - ActionHelper.editFirstValue(values, (s -> "%" + s)); - expectedNoOfParams = 1; - } - case NOT_CONTAINS -> - { - clause += " NOT LIKE ?"; - ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); - expectedNoOfParams = 1; - } - case LESS_THAN -> - { - clause += " < ?"; - expectedNoOfParams = 1; - } - case LESS_THAN_OR_EQUALS -> - { - clause += " <= ?"; - expectedNoOfParams = 1; - } - case GREATER_THAN -> - { - clause += " > ?"; - expectedNoOfParams = 1; - } - case GREATER_THAN_OR_EQUALS -> - { - clause += " >= ?"; - expectedNoOfParams = 1; - } - case IS_BLANK -> - { - clause += " IS NULL"; - if(field.getType().isStringLike()) - { - clause += " OR " + column + " = ''"; - } - expectedNoOfParams = 0; - } - case IS_NOT_BLANK -> - { - clause += " IS NOT NULL"; - if(field.getType().isStringLike()) - { - clause += " AND " + column + " != ''"; - } - expectedNoOfParams = 0; - } - case BETWEEN -> - { - clause += " BETWEEN ? AND ?"; - expectedNoOfParams = 2; - } - case NOT_BETWEEN -> - { - clause += " NOT BETWEEN ? AND ?"; - expectedNoOfParams = 2; - } - case TRUE -> - { - clause = " 1 = 1 "; - expectedNoOfParams = 0; - } - case FALSE -> - { - clause = " 0 = 1 "; - expectedNoOfParams = 0; - } - default -> throw new IllegalStateException("Unexpected operator: " + criterion.getOperator()); } + Integer expectedNoOfParams = actionStrategy.appendCriterionToWhereClause(criterion, clause, column, values, field); + if(expectedNoOfParams != null) { if(expectedNoOfParams.equals(1) && StringUtils.hasContent(criterion.getOtherFieldName())) @@ -652,7 +508,7 @@ public abstract class AbstractRDBMSAction JoinsContext.FieldAndTableNameOrAlias otherFieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(criterion.getOtherFieldName()); String otherColumn = escapeIdentifier(otherFieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(otherFieldAndTableNameOrAlias.field())); - clause = clause.replace("?", otherColumn); + clause = new StringBuilder(clause.toString().replace("?", otherColumn)); ///////////////////////////////////////////////////////////////////// // make sure we don't add any values in this case, just in case... // @@ -797,53 +653,7 @@ public abstract class AbstractRDBMSAction *******************************************************************************/ protected Serializable getFieldValueFromResultSet(QFieldType type, ResultSet resultSet, int i) throws SQLException { - switch(type) - { - case STRING: - case TEXT: - case HTML: - case PASSWORD: - { - return (QueryManager.getString(resultSet, i)); - } - case INTEGER: - { - return (QueryManager.getInteger(resultSet, i)); - } - case LONG: - { - return (QueryManager.getLong(resultSet, i)); - } - case DECIMAL: - { - return (QueryManager.getBigDecimal(resultSet, i)); - } - case DATE: - { - // todo - queryManager.getLocalDate? - return (QueryManager.getDate(resultSet, i)); - } - case TIME: - { - return (QueryManager.getLocalTime(resultSet, i)); - } - case DATE_TIME: - { - return (QueryManager.getInstant(resultSet, i)); - } - case BOOLEAN: - { - return (QueryManager.getBoolean(resultSet, i)); - } - case BLOB: - { - return (QueryManager.getByteArray(resultSet, i)); - } - default: - { - throw new IllegalStateException("Unexpected field type: " + type); - } - } + return (actionStrategy.getFieldValueFromResultSet(type, resultSet, i)); } @@ -1151,4 +961,26 @@ public abstract class AbstractRDBMSAction return (filter.clone()); } } + + + + /******************************************************************************* + ** Setter for backendMetaData + ** + *******************************************************************************/ + protected void setBackendMetaData(QBackendMetaData backendMetaData) + { + this.backendMetaData = (RDBMSBackendMetaData) backendMetaData; + this.actionStrategy = this.backendMetaData.getActionStrategy(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected RDBMSActionStrategyInterface getActionStrategy() + { + return (this.actionStrategy); + } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index 5c2f8710..e4d69585 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -48,7 +48,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; /******************************************************************************* @@ -70,6 +69,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega try { QTableMetaData table = aggregateInput.getTable(); + setBackendMetaData(aggregateInput.getBackend()); QQueryFilter filter = clonedOrNewFilter(aggregateInput.getFilter()); JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), table.getName(), aggregateInput.getQueryJoins(), filter); @@ -126,7 +126,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega queryStat.setStartTimestamp(Instant.now()); } - QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> + getActionStrategy().executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index 121c2c9b..2aac8fb2 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -40,7 +40,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import org.apache.commons.lang.BooleanUtils; @@ -63,6 +62,7 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf try { QTableMetaData table = countInput.getTable(); + setBackendMetaData(countInput.getBackend()); QQueryFilter filter = clonedOrNewFilter(countInput.getFilter()); JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), countInput.getTableName(), countInput.getQueryJoins(), filter); @@ -106,7 +106,7 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf queryStat.setStartTimestamp(Instant.now()); } - QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> + getActionStrategy().executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java index 9869435d..b59e9588 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java @@ -39,7 +39,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; /******************************************************************************* @@ -67,6 +66,8 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte *******************************************************************************/ public DeleteOutput execute(DeleteInput deleteInput) throws QException { + setBackendMetaData(deleteInput.getBackend()); + DeleteOutput deleteOutput = new DeleteOutput(); deleteOutput.setRecordsWithErrors(new ArrayList<>()); @@ -196,7 +197,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte try { - int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKey); + int rowCount = getActionStrategy().executeUpdateForRowCount(connection, sql, primaryKey); deleteOutput.addToDeletedRecordCount(rowCount); ///////////////////////////////////////////////////////////////////////////////////////////////////// @@ -249,7 +250,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte // todo sql customization - can edit sql and/or param list - Integer rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKeys); + Integer rowCount = getActionStrategy().executeUpdateForRowCount(connection, sql, primaryKeys); deleteOutput.addToDeletedRecordCount(rowCount); } catch(Exception e) @@ -287,7 +288,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte try { - int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, params); + int rowCount = getActionStrategy().executeUpdateForRowCount(connection, sql, params); deleteOutput.setDeletedRecordCount(rowCount); } catch(Exception e) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index 42ac344c..d2a78560 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -56,6 +56,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte { InsertOutput rs = new InsertOutput(); QTableMetaData table = insertInput.getTable(); + setBackendMetaData(insertInput.getBackend()); Connection connection = null; boolean needToCloseConnection = false; @@ -90,10 +91,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte needToCloseConnection = true; } - for(List page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE)) + for(List page : CollectionUtils.getPages(insertInput.getRecords(), getActionStrategy().getPageSize(insertInput))) { - String tableName = escapeIdentifier(getTableName(table)); - sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); + String backendTableName = escapeIdentifier(getTableName(table)); + sql = new StringBuilder("INSERT INTO ").append(backendTableName).append("(").append(columns).append(") VALUES"); params = new ArrayList<>(); int recordIndex = 0; @@ -132,6 +133,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(QRecord record : page) { QRecord outputRecord = new QRecord(record); + if(!StringUtils.hasContent(outputRecord.getTableName())) + { + outputRecord.setTableName(table.getName()); + } outputRecords.add(outputRecord); } continue; @@ -146,17 +151,24 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte // todo sql customization - can edit sql and/or param list // todo - non-serial-id style tables // todo - other generated values, e.g., createDate... maybe need to re-select? - List idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params, table.getField(table.getPrimaryKeyField()).getType()); + List idList = getActionStrategy().executeInsertForGeneratedIds(connection, sql.toString(), params, table.getField(table.getPrimaryKeyField())); int index = 0; for(QRecord record : page) { QRecord outputRecord = new QRecord(record); + if(!StringUtils.hasContent(outputRecord.getTableName())) + { + outputRecord.setTableName(table.getName()); + } outputRecords.add(outputRecord); if(CollectionUtils.nullSafeIsEmpty(record.getErrors())) { - Serializable id = idList.get(index++); - outputRecord.setValue(table.getPrimaryKeyField(), id); + if(idList.size() > index) + { + Serializable id = idList.get(index++); + outputRecord.setValue(table.getPrimaryKeyField(), id); + } } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 8ead329e..5c98b8c4 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -57,7 +57,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -94,6 +93,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf { QTableMetaData table = queryInput.getTable(); String tableName = queryInput.getTableName(); + setBackendMetaData(queryInput.getBackend()); List params = new ArrayList<>(); Selection selection = makeSelection(queryInput); @@ -140,7 +140,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf ////////////////////////////////////////////// QueryOutput queryOutput = new QueryOutput(queryInput); - QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> + getActionStrategy().executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java index 627b9c60..1894d911 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java @@ -41,7 +41,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; /******************************************************************************* @@ -66,6 +65,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte public UpdateOutput execute(UpdateInput updateInput) throws QException { QTableMetaData table = updateInput.getTable(); + setBackendMetaData(updateInput.getBackend()); UpdateActionRecordSplitHelper updateActionRecordSplitHelper = new UpdateActionRecordSplitHelper(); updateActionRecordSplitHelper.init(updateInput); @@ -181,7 +181,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte //////////////////////////////////////////////////////////////////////////////// try { - QueryManager.executeBatchUpdate(connection, sql, values); + getActionStrategy().executeBatchUpdate(connection, sql, values); incrementStatus(updateInput, recordList.size()); } finally @@ -214,7 +214,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte *******************************************************************************/ private void updateRecordsWithMatchingValuesAndFields(UpdateInput updateInput, Connection connection, QTableMetaData table, List recordList, List fieldsBeingUpdated) throws SQLException { - for(List page : CollectionUtils.getPages(recordList, QueryManager.PAGE_SIZE)) + for(List page : CollectionUtils.getPages(recordList, getActionStrategy().getPageSize(updateInput))) { ////////////////////////////// // skip records with errors // @@ -256,7 +256,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte ///////////////////////////////////// try { - QueryManager.executeUpdate(connection, sql, params); + getActionStrategy().executeUpdate(connection, sql, params); incrementStatus(updateInput, page.size()); } finally diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java new file mode 100644 index 00000000..87d26536 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java @@ -0,0 +1,122 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.jdbc; + + +import java.sql.Connection; +import java.sql.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.mchange.v2.c3p0.ConnectionCustomizer; + + +/******************************************************************************* + ** Basic version of a C3P0 Connection Customizer used by QQQ - that does things + ** expected for an RDBMS backend - specifically: + ** - runs queriesForNewConnections, if they are set. + *******************************************************************************/ +public class BaseC3P0ConnectionCustomizer implements ConnectionCustomizer +{ + private static final QLogger LOG = QLogger.getLogger(BaseC3P0ConnectionCustomizer.class); + + private static Map> queriesForNewConnections = new HashMap<>(); + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void onAcquire(Connection connection, String dataSourceIdentityToken) throws Exception + { + try + { + List queries = queriesForNewConnections.get(dataSourceIdentityToken); + if(CollectionUtils.nullSafeHasContents(queries)) + { + for(String sql : queries) + { + Statement statement = connection.createStatement(); + statement.execute(sql); + } + } + } + catch(Exception e) + { + LOG.warn("Exception on a query-for-new-connection", e); + throw (e); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void onDestroy(Connection connection, String dataSourceIdentityToken) throws Exception + { + ////////// + // noop // + ////////// + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void onCheckOut(Connection connection, String dataSourceIdentityToken) throws Exception + { + ////////// + // noop // + ////////// + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void onCheckIn(Connection connection, String dataSourceIdentityToken) throws Exception + { + ////////// + // noop // + ////////// + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public static void setQueriesForNewConnections(String backendName, List queriesForNewConnections) + { + BaseC3P0ConnectionCustomizer.queriesForNewConnections.put(backendName, queriesForNewConnections); + } + +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProvider.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProvider.java index 1a4740e4..e5ea2e2f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProvider.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProvider.java @@ -26,6 +26,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.util.LinkedHashMap; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.ConnectionPoolSettings; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.mchange.v2.c3p0.ComboPooledDataSource; @@ -110,6 +111,17 @@ public class C3P0PooledConnectionProvider implements ConnectionProviderInterface } } + pool.setIdentityToken(backend.getName()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the backend specifies queries to run for new connections, then set up a connection customizer to run them // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(backend.getQueriesForNewConnections())) + { + BaseC3P0ConnectionCustomizer.setQueriesForNewConnections(backend.getName(), backend.getQueriesForNewConnections()); + pool.setConnectionCustomizerClassName(BaseC3P0ConnectionCustomizer.class.getName()); + } + customizePool(pool); this.connectionPool = pool; diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java index 5ec62e34..0aa7d18d 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java @@ -178,7 +178,18 @@ public class ConnectionManager case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; case RDBMSBackendMetaData.VENDOR_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; case RDBMSBackendMetaData.VENDOR_H2 -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; - default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor()); + default -> + { + String connectionString = backend.buildConnectionString(); + if(connectionString == null) + { + throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor() + " (and null returned by backendMetaData.buildConnectionString())"); + } + else + { + yield (connectionString); + } + } }; } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index ceed311a..72069193 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -22,7 +22,6 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; -import java.io.Serializable; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Connection; @@ -31,7 +30,6 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; @@ -43,7 +41,6 @@ import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; @@ -53,16 +50,14 @@ import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.apache.commons.lang.NotImplementedException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* - ** + ** Note that much of this class is/was ported (well, copied) to BaseRDBMSActionStrategy + ** around 2025-01, during the addition of SQLite backend module. *******************************************************************************/ public class QueryManager { @@ -169,49 +164,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeStatementForeachResult(Connection connection, String sql, ResultSetProcessor processor, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - PreparedStatement statement = null; - ResultSet resultSet = null; - - try - { - if(params.length == 1 && params[0] instanceof Collection) - { - params = ((Collection) params[0]).toArray(); - } - - statement = prepareStatementAndBindParams(connection, sql, params); - statement.execute(); - resultSet = statement.getResultSet(); - - while(resultSet.next()) - { - processor.processResultSet(resultSet); - } - } - finally - { - if(statement != null) - { - statement.close(); - } - - if(resultSet != null) - { - resultSet.close(); - } - } - */ - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -269,57 +221,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static Map executeStatementForSingleRow(Connection connection, String sql, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - statement.execute(); - ResultSet resultSet = statement.getResultSet(); - if(resultSet.next()) - { - Map rs = new LinkedHashMap<>(); - - ResultSetMetaData metaData = resultSet.getMetaData(); - for(int i = 1; i <= metaData.getColumnCount(); i++) - { - rs.put(metaData.getColumnName(i), getObject(resultSet, i)); - } - - return (rs); - } - else - { - return (null); - } - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static SimpleEntity executeStatementForSimpleEntity(Connection connection, String sql, Object... params) throws SQLException - { - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - statement.execute(); - ResultSet resultSet = statement.getResultSet(); - if(resultSet.next()) - { - return (buildSimpleEntity(resultSet)); - } - else - { - return (null); - } - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -347,48 +248,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static List executeStatementForSimpleEntityList(Connection connection, String sql, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - List rs = new ArrayList<>(); - - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - statement.execute(); - ResultSet resultSet = statement.getResultSet(); - while(resultSet.next()) - { - SimpleEntity row = buildSimpleEntity(resultSet); - - rs.add(row); - } - - return (rs); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static SimpleEntity buildSimpleEntity(ResultSet resultSet) throws SQLException - { - SimpleEntity row = new SimpleEntity(); - - ResultSetMetaData metaData = resultSet.getMetaData(); - for(int i = 1; i <= metaData.getColumnCount(); i++) - { - row.put(metaData.getColumnName(i), getObject(resultSet, i)); - } - return row; - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -429,38 +288,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeUpdateVoid(Connection connection, String sql, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) - { - statement.executeUpdate(); - } - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeUpdateVoid(Connection connection, String sql, List params) throws SQLException - { - throw (new NotImplementedException()); - /* - try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) - { - statement.executeUpdate(); - } - */ - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -481,189 +308,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer executeUpdateForRowCount(Connection connection, String sql, List params) throws SQLException - { - throw (new NotImplementedException()); - /* - try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) - { - statement.executeUpdate(); - return (statement.getUpdateCount()); - } - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer executeInsertForGeneratedId(Connection connection, String sql, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) - { - bindParams(params, statement); - statement.executeUpdate(); - ResultSet generatedKeys = statement.getGeneratedKeys(); - if(generatedKeys.next()) - { - return (getInteger(generatedKeys, 1)); - } - else - { - return (null); - } - } - */ - } - - - - /******************************************************************************* - ** todo - needs (specific) unit test - *******************************************************************************/ - public static List executeInsertForGeneratedIds(Connection connection, String sql, List params, QFieldType idType) throws SQLException - { - try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) - { - bindParams(params.toArray(), statement); - incrementStatistic(STAT_QUERIES_RAN); - statement.executeUpdate(); - - ///////////////////////////////////////////////////////////// - // We default to idType of INTEGER if it was not passed in // - ///////////////////////////////////////////////////////////// - if(idType == null) - { - idType = QFieldType.INTEGER; - } - - ResultSet generatedKeys = statement.getGeneratedKeys(); - List rs = new ArrayList<>(); - while(generatedKeys.next()) - { - switch(idType) - { - case INTEGER: - { - rs.add(getInteger(generatedKeys, 1)); - break; - } - case LONG: - { - rs.add(getLong(generatedKeys, 1)); - break; - } - default: - { - LOG.warn("Unknown id data type, attempting to getInteger.", logPair("sql", sql)); - rs.add(getInteger(generatedKeys, 1)); - break; - } - } - } - return (rs); - } - catch(SQLException e) - { - LOG.warn("SQLException", e, logPair("sql", sql)); - throw (e); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeInsertForList(Connection connection, List entityList) throws SQLException - { - throw (new NotImplementedException()); - /* - List> pages = CollectionUtils.getPages(entityList, PAGE_SIZE); - for(List page : pages) - { - ArrayList columns = new ArrayList<>(page.get(0).keySet()); - String sql = "INSERT INTO " + page.get(0).getTableName() + "(" + StringUtils.join(",", columns) + ") VALUES (" + columns.stream().map(s -> "?").collect(Collectors.joining(",")) + ")"; - - PreparedStatement insertPS = connection.prepareStatement(sql); - for(SimpleEntity entity : page) - { - Object[] params = new Object[columns.size()]; - for(int i = 0; i < columns.size(); i++) - { - params[i] = entity.get(columns.get(i)); - } - - bindParams(insertPS, params); - insertPS.addBatch(); - } - insertPS.executeBatch(); - } - - for(List page : pages) - { - page.clear(); - } - pages.clear(); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer executeInsert(Connection connection, SimpleEntity entity) throws SQLException - { - throw (new NotImplementedException()); - /* - ArrayList columns = new ArrayList<>(entity.keySet()); - String sql = "INSERT INTO " + entity.getTableName() + "(" + StringUtils.join(",", columns) + ") VALUES (" + columns.stream().map(s -> "?").collect(Collectors.joining(",")) + ")"; - - Object[] params = new Object[columns.size()]; - for(int i = 0; i < columns.size(); i++) - { - params[i] = entity.get(columns.get(i)); - } - - return (executeInsertForGeneratedId(connection, sql, params)); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeBatchUpdate(Connection connection, String updateSQL, List> values) throws SQLException - { - for(List> page : CollectionUtils.getPages(values, PAGE_SIZE)) - { - PreparedStatement updatePS = connection.prepareStatement(updateSQL); - for(List row : page) - { - Object[] params = new Object[row.size()]; - for(int i = 0; i < row.size(); i++) - { - params[i] = row.get(i); - } - - bindParams(updatePS, params); - updatePS.addBatch(); - } - incrementStatistic(STAT_BATCHES_RAN); - updatePS.executeBatch(); - } - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -734,15 +378,8 @@ public class QueryManager /******************************************************************************* * index is 1-based!! *******************************************************************************/ - @SuppressWarnings("unchecked") public static int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException { - /* if(value instanceof TypeValuePair) - { - bindParamTypeValuePair(statement, index, (TypeValuePair) value); - return (1); - } - else*/ if(value instanceof Integer i) { bindParam(statement, index, i); @@ -856,68 +493,6 @@ public class QueryManager } } - /******************************************************************************* - ** - *******************************************************************************/ - /* - public static TypeValuePair param(Class c, T v) - { - return (new TypeValuePair<>(c, v)); - } - */ - - /******************************************************************************* - ** - *******************************************************************************/ - /* - private static void bindParamTypeValuePair(PreparedStatement statement, int index, TypeValuePair value) throws SQLException - { - Object v = value.getValue(); - Class t = value.getType(); - - if(t.equals(Integer.class)) - { - bindParam(statement, index, (Integer) v); - } - else if(t.equals(String.class)) - { - bindParam(statement, index, (String) v); - } - else if(t.equals(Boolean.class)) - { - bindParam(statement, index, (Boolean) v); - } - else if(t.equals(Timestamp.class)) - { - bindParam(statement, index, (Timestamp) v); - } - else if(t.equals(Date.class)) - { - bindParam(statement, index, (Date) v); - } - else if(t.equals(Calendar.class)) - { - bindParam(statement, index, (Calendar) v); - } - else if(t.equals(LocalDate.class)) - { - bindParam(statement, index, (LocalDate) v); - } - else if(t.equals(LocalDateTime.class)) - { - bindParam(statement, index, (LocalDateTime) v); - } - else if(t.equals(BigDecimal.class)) - { - bindParam(statement, index, (BigDecimal) v); - } - else - { - throw (new SQLException("Unexpected value type [" + t.getSimpleName() + "] in bindParamTypeValuePair.")); - } - } - */ - /******************************************************************************* @@ -1675,170 +1250,6 @@ public class QueryManager - /******************************************************************************* - ** Find an id from a "large" table that was created X days ago (assumes the date - ** field in the table isn't indexed, but id is - so do a binary search on id, - ** selecting the date of the min & max & mid id, then sub-dividing until the goal - ** days-ago is found). - ** - *******************************************************************************/ - public static Integer findIdForDaysAgo(Connection connection, String tableName, String dateFieldName, int goalDaysAgo) throws SQLException - { - throw (new NotImplementedException()); - /* - return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalDaysAgo, ChronoUnit.DAYS)); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer findIdForTimestamp(Connection connection, String tableName, String dateFieldName, LocalDateTime timestamp) throws SQLException - { - throw (new NotImplementedException()); - /* - long between = ChronoUnit.SECONDS.between(timestamp, LocalDateTime.now()); - return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, (int) between, ChronoUnit.SECONDS)); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer findIdForTimeUnitAgo(Connection connection, String tableName, String dateFieldName, int goalUnitsAgo, ChronoUnit unit) throws SQLException - { - throw (new NotImplementedException()); - /* - Integer maxId = executeStatementForSingleValue(connection, Integer.class, "SELECT MAX(id) FROM " + tableName); - Integer minId = executeStatementForSingleValue(connection, Integer.class, "SELECT MIN(id) FROM " + tableName); - - if(maxId == null || minId == null) - { - // Logger.logDebug("For [" + tableName + "], returning null id for X time-units ago, because either a min or max wasn't found."); - return (null); - } - - Integer idForGoal = findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalUnitsAgo, minId, maxId, unit); - long foundUnitsAgo = getTimeUnitAgo(connection, tableName, dateFieldName, idForGoal, unit); - - // Logger.logDebug("For [" + tableName + "], using min id [" + idForGoal + "], which is from [" + foundUnitsAgo + "] Units[" + unit + "] ago."); - - return (idForGoal); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static Integer findIdForTimeUnitAgo(Connection connection, String tableName, String dateFieldName, int goalUnitsAgo, Integer minId, Integer maxId, ChronoUnit unit) throws SQLException - { - throw (new NotImplementedException()); - /* - Integer midId = minId + ((maxId - minId) / 2); - if(midId.equals(minId) || midId.equals(maxId)) - { - return (midId); - } - - long foundUnitsAgo = getTimeUnitAgo(connection, tableName, dateFieldName, midId, unit); - if(foundUnitsAgo == goalUnitsAgo) - { - return (midId); - } - else if(foundUnitsAgo > goalUnitsAgo) - { - return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalUnitsAgo, midId, maxId, unit)); - } - else - { - return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalUnitsAgo, minId, midId, unit)); - } - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static long getTimeUnitAgo(Connection connection, String tableName, String dateFieldName, Integer id, ChronoUnit unit) throws SQLException - { - throw (new NotImplementedException()); - /* - LocalDateTime now = LocalDateTime.now(); - - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // note, we used to just do where id=? here - but if that row is ever missing, we have a bad time - so - do id >= ? order by id, and just the first row. // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - LocalDateTime date = executeStatementForSingleValue(connection, LocalDateTime.class, "SELECT " + dateFieldName + " FROM " + tableName + " WHERE id >= ? ORDER BY id LIMIT 1", id); - // System.out.println(date); - - // if(date == null) - { - // return now. - } - // else - { - long diff = unit.between(date, now); - // System.out.println("Unit[" + unit + "]'s ago: " + diff); - return diff; - } - */ - } - - /******************************************************************************* - ** - *******************************************************************************/ - // public static class TypeValuePair - // { - // private Class type; - // private T value; - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // @SuppressWarnings("unchecked") - // public TypeValuePair(T value) - // { - // this.value = value; - // this.type = (Class) value.getClass(); - // } - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public TypeValuePair(Class type, T value) - // { - // this.type = type; - // this.value = value; - // } - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public T getValue() - // { - // return (value); - // } - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public Class getType() - // { - // return (type); - // } - - // } - - - /******************************************************************************* ** Setter for collectStatistics ** diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleConnectionProvider.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleConnectionProvider.java index 264f5528..870da925 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleConnectionProvider.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleConnectionProvider.java @@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import static com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager.getJdbcUrl; @@ -57,7 +59,28 @@ public class SimpleConnectionProvider implements ConnectionProviderInterface public Connection getConnection() throws SQLException { String jdbcURL = getJdbcUrl(backend); - return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); + Connection connection = DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); + + if(CollectionUtils.nullSafeHasContents(backend.getQueriesForNewConnections())) + { + runQueriesForNewConnections(connection); + } + + return (connection); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private void runQueriesForNewConnections(Connection connection) throws SQLException + { + for(String sql : backend.getQueriesForNewConnections()) + { + Statement statement = connection.createStatement(); + statement.execute(sql); + } } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleEntity.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleEntity.java deleted file mode 100755 index 8541d794..00000000 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleEntity.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.module.rdbms.jdbc; - - -import java.util.HashMap; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class SimpleEntity extends HashMap -{ - // private String tableName; - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public SimpleEntity() - // { - // super(); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public SimpleEntity with(String key, Object value) - // { - // put(key, value); - // return (this); - // } - - - - // /******************************************************************************* - // ** Return the current value of tableName - // ** - // ** @return tableName - // *******************************************************************************/ - // public String getTableName() - // { - // return (tableName); - // } - - - - // /******************************************************************************* - // ** Set the current value of tableName - // ** - // ** @param tableName - // *******************************************************************************/ - // public void setTableName(String tableName) - // { - // this.tableName = tableName; - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public SimpleEntity withTableName(String tableName) - // { - // setTableName(tableName); - // return (this); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public Boolean getBoolean(String columnName) - // { - // Object o = get(columnName); - // if(o == null) - // { - // return (null); - // } - - // if(o instanceof Boolean) - // { - // return ((Boolean) o); - // } - // else if(o instanceof Number) - // { - // int i = ((Number) o).intValue(); - // return (i != 0); - // } - // else if(o instanceof String) - // { - // String s = (String) o; - // return (s.equalsIgnoreCase("1") || s.equalsIgnoreCase("true") || s.equalsIgnoreCase("t")); - // } - // else - // { - // throw new IllegalArgumentException("Could not get value of object of type [" + o.getClass() + "] as Boolean."); - // } - - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public String getString(String columnName) - // { - // Object o = get(columnName); - // if(o == null) - // { - // return (null); - // } - // if(o instanceof String) - // { - // return ((String) o); - // } - // else if(o instanceof byte[]) - // { - // return (new String((byte[]) o)); - // } - - // return String.valueOf(o); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public Integer getInteger(String columnName) - // { - // Object o = get(columnName); - // if(o instanceof Long) - // { - // return ((Long) o).intValue(); - // } - // else if(o instanceof Short) - // { - // return ((Short) o).intValue(); - // } - // else if(o instanceof String) - // { - // return (Integer.parseInt((String) o)); - // } - - // return ((Integer) o); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public BigDecimal getBigDecimal(String columnName) - // { - // Object o = get(columnName); - // if(o == null) - // { - // return (null); - // } - - // if(o instanceof BigDecimal) - // { - // return ((BigDecimal) o); - // } - // else - // { - // return new BigDecimal(String.valueOf(o)); - // } - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public Long getLong(String columnName) - // { - // Object o = get(columnName); - // if(o instanceof Integer) - // { - // return ((Integer) o).longValue(); - // } - - // return ((Long) o); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public void trimStrings() - // { - // for(String key : keySet()) - // { - // Object value = get(key); - // if(value != null && value instanceof String) - // { - // put(key, ((String) value).trim()); - // } - // } - // } -} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java index 110db12d..277502ed 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java @@ -22,10 +22,15 @@ package com.kingsrook.qqq.backend.module.rdbms.model.metadata; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; /******************************************************************************* @@ -49,6 +54,11 @@ public class RDBMSBackendMetaData extends QBackendMetaData private RDBMSBackendMetaData readOnlyBackendMetaData; + private QCodeReference actionStrategyCodeReference; + private RDBMSActionStrategyInterface actionStrategy; + + private List queriesForNewConnections = null; + /////////////////////////////////////////////////////////// // define well-known (and fully supported) vendor values // /////////////////////////////////////////////////////////// @@ -453,4 +463,121 @@ public class RDBMSBackendMetaData extends QBackendMetaData return (this); } + + + /*************************************************************************** + ** + ***************************************************************************/ + public String buildConnectionString() + { + return null; + } + + + + /******************************************************************************* + ** Getter for actionStrategyCodeReference + *******************************************************************************/ + public QCodeReference getActionStrategyCodeReference() + { + return (this.actionStrategyCodeReference); + } + + + + /******************************************************************************* + ** Setter for actionStrategyCodeReference + *******************************************************************************/ + public void setActionStrategyCodeReference(QCodeReference actionStrategyCodeReference) + { + this.actionStrategyCodeReference = actionStrategyCodeReference; + } + + + + /******************************************************************************* + ** Fluent setter for actionStrategyCodeReference + *******************************************************************************/ + public RDBMSBackendMetaData withActionStrategyCodeReference(QCodeReference actionStrategyCodeReference) + { + this.actionStrategyCodeReference = actionStrategyCodeReference; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public RDBMSActionStrategyInterface getActionStrategy() + { + if(actionStrategy == null) + { + if(actionStrategyCodeReference != null) + { + actionStrategy = QCodeLoader.getAdHoc(RDBMSActionStrategyInterface.class, actionStrategyCodeReference); + } + else + { + actionStrategy = new BaseRDBMSActionStrategy(); + } + } + + return (actionStrategy); + } + + + + /*************************************************************************** + * note - protected - meant for sub-classes to use in their implementation of + * getActionStrategy, but not for public use. + ***************************************************************************/ + protected RDBMSActionStrategyInterface getActionStrategyField() + { + return (actionStrategy); + } + + + + /*************************************************************************** + * note - protected - meant for sub-classes to use in their implementation of + * getActionStrategy, but not for public use. + ***************************************************************************/ + protected void setActionStrategyField(RDBMSActionStrategyInterface actionStrategy) + { + this.actionStrategy = actionStrategy; + } + + + /******************************************************************************* + ** Getter for queriesForNewConnections + *******************************************************************************/ + public List getQueriesForNewConnections() + { + return (this.queriesForNewConnections); + } + + + + /******************************************************************************* + ** Setter for queriesForNewConnections + *******************************************************************************/ + public void setQueriesForNewConnections(List queriesForNewConnections) + { + this.queriesForNewConnections = queriesForNewConnections; + } + + + + /******************************************************************************* + ** Fluent setter for queriesForNewConnections + *******************************************************************************/ + public RDBMSBackendMetaData withQueriesForNewConnections(List queriesForNewConnections) + { + this.queriesForNewConnections = queriesForNewConnections; + return (this); + } + + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSFieldMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSFieldMetaData.java new file mode 100644 index 00000000..58e6fca4 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSFieldMetaData.java @@ -0,0 +1,166 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QSupplementalFieldMetaData; +import com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule; +import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RDBMSFieldMetaData extends QSupplementalFieldMetaData +{ + private QCodeReference actionStrategyCodeReference; + private RDBMSActionStrategyInterface actionStrategy; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RDBMSFieldMetaData() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static RDBMSFieldMetaData of(QFieldMetaData field) + { + return ((RDBMSFieldMetaData) field.getSupplementalMetaData(RDBMSBackendModule.NAME)); + } + + + + /******************************************************************************* + ** either get the object attached to a field - or create a new one and attach + ** it to the field, and return that. + *******************************************************************************/ + public static RDBMSFieldMetaData ofOrWithNew(QFieldMetaData field) + { + RDBMSFieldMetaData rdbmsFieldMetaData = of(field); + if(rdbmsFieldMetaData == null) + { + rdbmsFieldMetaData = new RDBMSFieldMetaData(); + field.withSupplementalMetaData(rdbmsFieldMetaData); + } + return (rdbmsFieldMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getType() + { + return (RDBMSBackendModule.NAME); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public RDBMSActionStrategyInterface getActionStrategy() + { + if(actionStrategy == null) + { + if(actionStrategyCodeReference != null) + { + actionStrategy = QCodeLoader.getAdHoc(RDBMSActionStrategyInterface.class, actionStrategyCodeReference); + } + else + { + return (null); + } + } + + return (actionStrategy); + } + + + + /******************************************************************************* + ** Getter for actionStrategyCodeReference + *******************************************************************************/ + public QCodeReference getActionStrategyCodeReference() + { + return (this.actionStrategyCodeReference); + } + + + + /******************************************************************************* + ** Setter for actionStrategyCodeReference + *******************************************************************************/ + public void setActionStrategyCodeReference(QCodeReference actionStrategyCodeReference) + { + this.actionStrategyCodeReference = actionStrategyCodeReference; + } + + + + /******************************************************************************* + ** Fluent setter for actionStrategyCodeReference + *******************************************************************************/ + public RDBMSFieldMetaData withActionStrategyCodeReference(QCodeReference actionStrategyCodeReference) + { + this.actionStrategyCodeReference = actionStrategyCodeReference; + return (this); + } + + + + /*************************************************************************** + * note - protected - meant for sub-classes to use in their implementation of + * getActionStrategy, but not for public use. + ***************************************************************************/ + protected RDBMSActionStrategyInterface getActionStrategyField() + { + return (actionStrategy); + } + + + + /*************************************************************************** + * note - protected - meant for sub-classes to use in their implementation of + * getActionStrategy, but not for public use. + ***************************************************************************/ + protected void setActionStrategyField(RDBMSActionStrategyInterface actionStrategy) + { + this.actionStrategy = actionStrategy; + } + +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java new file mode 100644 index 00000000..5bde3252 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java @@ -0,0 +1,873 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Note that much of this class came from the old (old) QueryManager class. + *******************************************************************************/ +public class BaseRDBMSActionStrategy implements RDBMSActionStrategyInterface +{ + private static final QLogger LOG = QLogger.getLogger(BaseRDBMSActionStrategy.class); + + private static final int MILLIS_PER_SECOND = 1000; + + public static final int DEFAULT_PAGE_SIZE = 2000; + public static int PAGE_SIZE = DEFAULT_PAGE_SIZE; + + private boolean collectStatistics = false; + private final Map statistics = Collections.synchronizedMap(new HashMap<>()); + + public static final String STAT_QUERIES_RAN = "queriesRan"; + public static final String STAT_BATCHES_RAN = "batchesRan"; + + + + /*************************************************************************** + * + ***************************************************************************/ + public Integer appendCriterionToWhereClause(QFilterCriteria criterion, StringBuilder clause, String column, List values, QFieldMetaData field) + { + clause.append(column); + + switch(criterion.getOperator()) + { + case EQUALS -> + { + clause.append(" = ?"); + return (1); + } + case NOT_EQUALS -> + { + clause.append(" != ?"); + return (1); + } + case NOT_EQUALS_OR_IS_NULL -> + { + clause.append(" != ? OR ").append(column).append(" IS NULL "); + return (1); + } + case IN -> + { + if(values.isEmpty()) + { + /////////////////////////////////////////////////////// + // if there are no values, then we want a false here // + /////////////////////////////////////////////////////// + clause.delete(0, clause.length()); + clause.append(" 0 = 1 "); + return (0); + } + else + { + clause.append(" IN (").append(values.stream().map(x -> "?").collect(Collectors.joining(","))).append(")"); + return (values.size()); + } + } + case IS_NULL_OR_IN -> + { + clause.append(" IS NULL "); + + if(!values.isEmpty()) + { + clause.append(" OR ").append(column).append(" IN (").append(values.stream().map(x -> "?").collect(Collectors.joining(","))).append(")"); + return (values.size()); + } + else + { + return (0); + } + } + case NOT_IN -> + { + if(values.isEmpty()) + { + ////////////////////////////////////////////////////// + // if there are no values, then we want a true here // + ////////////////////////////////////////////////////// + clause.delete(0, clause.length()); + clause.append(" 1 = 1 "); + return (0); + } + else + { + clause.append(" NOT IN (").append(values.stream().map(x -> "?").collect(Collectors.joining(","))).append(")"); + return (values.size()); + } + } + case LIKE -> + { + clause.append(" LIKE ?"); + return (1); + } + case NOT_LIKE -> + { + clause.append(" NOT LIKE ?"); + return (1); + } + case STARTS_WITH -> + { + clause.append(" LIKE ?"); + ActionHelper.editFirstValue(values, (s -> s + "%")); + return (1); + } + case ENDS_WITH -> + { + clause.append(" LIKE ?"); + ActionHelper.editFirstValue(values, (s -> "%" + s)); + return (1); + } + case CONTAINS -> + { + clause.append(" LIKE ?"); + ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); + return (1); + } + case NOT_STARTS_WITH -> + { + clause.append(" NOT LIKE ?"); + ActionHelper.editFirstValue(values, (s -> s + "%")); + return (1); + } + case NOT_ENDS_WITH -> + { + clause.append(" NOT LIKE ?"); + ActionHelper.editFirstValue(values, (s -> "%" + s)); + return (1); + } + case NOT_CONTAINS -> + { + clause.append(" NOT LIKE ?"); + ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); + return (1); + } + case LESS_THAN -> + { + clause.append(" < ?"); + return (1); + } + case LESS_THAN_OR_EQUALS -> + { + clause.append(" <= ?"); + return (1); + } + case GREATER_THAN -> + { + clause.append(" > ?"); + return (1); + } + case GREATER_THAN_OR_EQUALS -> + { + clause.append(" >= ?"); + return (1); + } + case IS_BLANK -> + { + clause.append(" IS NULL"); + if(field.getType().isStringLike()) + { + clause.append(" OR ").append(column).append(" = ''"); + } + return (0); + } + case IS_NOT_BLANK -> + { + clause.append(" IS NOT NULL"); + if(field.getType().isStringLike()) + { + clause.append(" AND ").append(column).append(" != ''"); + } + return (0); + } + case BETWEEN -> + { + clause.append(" BETWEEN ? AND ?"); + return (2); + } + case NOT_BETWEEN -> + { + clause.append(" NOT BETWEEN ? AND ?"); + return (2); + } + case TRUE -> + { + clause.delete(0, clause.length()); + clause.append(" 1 = 1 "); + return (0); + } + case FALSE -> + { + clause.delete(0, clause.length()); + clause.append(" 0 = 1 "); + return (0); + } + default -> throw new IllegalStateException("Unexpected operator: " + criterion.getOperator()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Serializable getFieldValueFromResultSet(QFieldType type, ResultSet resultSet, int i) throws SQLException + { + return switch(type) + { + case STRING, TEXT, HTML, PASSWORD -> (QueryManager.getString(resultSet, i)); + case INTEGER -> (QueryManager.getInteger(resultSet, i)); + case LONG -> (QueryManager.getLong(resultSet, i)); + case DECIMAL -> (QueryManager.getBigDecimal(resultSet, i)); + case DATE -> (QueryManager.getDate(resultSet, i));// todo - queryManager.getLocalDate? + case TIME -> (QueryManager.getLocalTime(resultSet, i)); + case DATE_TIME -> (QueryManager.getInstant(resultSet, i)); + case BOOLEAN -> (QueryManager.getBoolean(resultSet, i)); + case BLOB -> (QueryManager.getByteArray(resultSet, i)); + default -> throw new IllegalStateException("Unexpected field type: " + type); + }; + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public PreparedStatement executeUpdate(Connection connection, String sql, List params) throws SQLException + { + try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, new List[] { params })) + { + incrementStatistic(STAT_QUERIES_RAN); + statement.executeUpdate(); + return (statement); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void executeBatchUpdate(Connection connection, String updateSQL, List> values) throws SQLException + { + for(List> page : CollectionUtils.getPages(values, PAGE_SIZE)) + { + PreparedStatement updatePS = connection.prepareStatement(updateSQL); + for(List row : page) + { + Object[] params = new Object[row.size()]; + for(int i = 0; i < row.size(); i++) + { + params[i] = row.get(i); + } + + bindParams(params, updatePS); + updatePS.addBatch(); + } + incrementStatistic(STAT_BATCHES_RAN); + updatePS.executeBatch(); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public List executeInsertForGeneratedIds(Connection connection, String sql, List params, QFieldMetaData primaryKeyField) throws SQLException + { + try(PreparedStatement statement = connection.prepareStatement(sql, new String[] { getColumnName(primaryKeyField) })) + { + bindParams(params.toArray(), statement); + incrementStatistic(STAT_QUERIES_RAN); + statement.executeUpdate(); + + ResultSet generatedKeys = statement.getGeneratedKeys(); + List rs = new ArrayList<>(); + while(generatedKeys.next()) + { + rs.add(getFieldValueFromResultSet(primaryKeyField.getType(), generatedKeys, 1)); + } + return (rs); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Integer executeUpdateForRowCount(Connection connection, String sql, Object... params) throws SQLException + { + try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) + { + incrementStatistic(STAT_QUERIES_RAN); + int rowCount = statement.executeUpdate(); + return (rowCount); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void executeStatement(PreparedStatement statement, CharSequence sql, ResultSetProcessor processor, Object... params) throws SQLException, QException + { + ResultSet resultSet = null; + + try + { + bindParams(params, statement); + incrementStatistic(STAT_QUERIES_RAN); + statement.execute(); + resultSet = statement.getResultSet(); + + if(processor != null) + { + processor.processResultSet(resultSet); + } + } + catch(SQLException e) + { + if(sql != null) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + } + throw (e); + } + finally + { + if(resultSet != null) + { + resultSet.close(); + } + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Integer getPageSize(AbstractActionInput actionInput) + { + return PAGE_SIZE; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected PreparedStatement prepareStatementAndBindParams(Connection connection, String sql, Object[] params) throws SQLException + { + PreparedStatement statement = connection.prepareStatement(sql); + bindParams(params, statement); + return statement; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void bindParams(Object[] params, PreparedStatement statement) throws SQLException + { + int paramIndex = 0; + if(params != null) + { + for(Object param : params) + { + int paramsBound = bindParamObject(statement, (paramIndex + 1), param); + paramIndex += paramsBound; + } + } + } + + + + /******************************************************************************* + * index is 1-based!! + *******************************************************************************/ + protected int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException + { + if(value instanceof Integer i) + { + bindParam(statement, index, i); + return (1); + } + else if(value instanceof Short s) + { + bindParam(statement, index, s.intValue()); + return (1); + } + else if(value instanceof Long l) + { + bindParam(statement, index, l); + return (1); + } + else if(value instanceof Double d) + { + bindParam(statement, index, d); + return (1); + } + else if(value instanceof String s) + { + bindParam(statement, index, s); + return (1); + } + else if(value instanceof Boolean b) + { + bindParam(statement, index, b); + return (1); + } + else if(value instanceof Timestamp ts) + { + bindParam(statement, index, ts); + return (1); + } + else if(value instanceof Date) + { + bindParam(statement, index, (Date) value); + return (1); + } + else if(value instanceof Calendar c) + { + bindParam(statement, index, c); + return (1); + } + else if(value instanceof BigDecimal bd) + { + bindParam(statement, index, bd); + return (1); + } + else if(value == null) + { + statement.setNull(index, Types.CHAR); + return (1); + } + else if(value instanceof Collection c) + { + int paramsBound = 0; + for(Object o : c) + { + paramsBound += bindParamObject(statement, (index + paramsBound), o); + } + return (paramsBound); + } + else if(value instanceof byte[] ba) + { + statement.setBytes(index, ba); + return (1); + } + else if(value instanceof Instant i) + { + statement.setObject(index, i); + return (1); + } + else if(value instanceof LocalDate ld) + { + @SuppressWarnings("deprecation") + Date date = new Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); + statement.setDate(index, date); + return (1); + } + else if(value instanceof LocalTime lt) + { + @SuppressWarnings("deprecation") + Time time = new Time(lt.getHour(), lt.getMinute(), lt.getSecond()); + statement.setTime(index, time); + return (1); + } + else if(value instanceof OffsetDateTime odt) + { + long epochMillis = odt.toEpochSecond() * MILLIS_PER_SECOND; + Timestamp timestamp = new Timestamp(epochMillis); + statement.setTimestamp(index, timestamp); + return (1); + } + else if(value instanceof LocalDateTime ldt) + { + ZoneOffset offset = OffsetDateTime.now().getOffset(); + long epochMillis = ldt.toEpochSecond(offset) * MILLIS_PER_SECOND; + Timestamp timestamp = new Timestamp(epochMillis); + statement.setTimestamp(index, timestamp); + return (1); + } + else if(value instanceof PossibleValueEnum pve) + { + return (bindParamObject(statement, index, pve.getPossibleValueId())); + } + else + { + throw (new SQLException("Unexpected value type [" + value.getClass().getSimpleName() + "] in bindParamObject.")); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Integer value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.INTEGER); + } + else + { + statement.setInt(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Long value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.INTEGER); + } + else + { + statement.setLong(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Double value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DOUBLE); + } + else + { + statement.setDouble(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, String value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.CHAR); + } + else + { + statement.setString(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Boolean value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.BOOLEAN); + } + else + { + statement.setBoolean(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Date value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DATE); + } + else + { + statement.setDate(index, new Date(value.getTime())); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Timestamp value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.TIMESTAMP); + } + else + { + statement.setTimestamp(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Calendar value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DATE); + } + else + { + statement.setTimestamp(index, new Timestamp(value.getTimeInMillis())); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, LocalDate value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DATE); + } + else + { + LocalDateTime localDateTime = value.atTime(0, 0); + Timestamp timestamp = new Timestamp(localDateTime.atZone(ZoneId.systemDefault()).toEpochSecond() * MILLIS_PER_SECOND); // TimeStamp expects millis, not seconds, after epoch + statement.setTimestamp(index, timestamp); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, LocalDateTime value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.TIMESTAMP); + } + else + { + Timestamp timestamp = new Timestamp(value.atZone(ZoneId.systemDefault()).toEpochSecond() * MILLIS_PER_SECOND); // TimeStamp expects millis, not seconds, after epoch + statement.setTimestamp(index, timestamp); + } + } + + + + /******************************************************************************* + ** + ** + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, BigDecimal value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DECIMAL); + } + else + { + statement.setBigDecimal(index, value); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, byte[] value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.ARRAY); + } + else + { + statement.setBytes(index, value); + } + } + + + + /******************************************************************************* + ** + ** + *******************************************************************************/ + protected void bindParamNull(PreparedStatement statement, int index) throws SQLException + { + statement.setNull(index, Types.NULL); + } + + + + /******************************************************************************* + ** Increment a statistic + ** + *******************************************************************************/ + protected void incrementStatistic(String statName) + { + if(collectStatistics) + { + statistics.putIfAbsent(statName, 0); + statistics.put(statName, statistics.get(statName) + 1); + } + } + + + + /******************************************************************************* + ** Setter for collectStatistics + ** + *******************************************************************************/ + public void setCollectStatistics(boolean collectStatistics) + { + this.collectStatistics = collectStatistics; + } + + + + /******************************************************************************* + ** clear the map of statistics + ** + *******************************************************************************/ + public void resetStatistics() + { + statistics.clear(); + } + + + + /******************************************************************************* + ** Getter for statistics + ** + *******************************************************************************/ + public Map getStatistics() + { + return statistics; + } + + + + /******************************************************************************* + ** Setter for pageSize + ** + *******************************************************************************/ + public void setPageSize(int pageSize) + { + BaseRDBMSActionStrategy.PAGE_SIZE = pageSize; + } + + /******************************************************************************* + ** Get the column name to use for a field in the RDBMS, from the fieldMetaData. + ** + ** That is, field.backendName if set -- else, field.name + *******************************************************************************/ + protected String getColumnName(QFieldMetaData field) + { + if(field.getBackendName() != null) + { + return (field.getBackendName()); + } + return (field.getName()); + } + +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategy.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategy.java new file mode 100644 index 00000000..f49ccf04 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategy.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** RDBMS action strategy for a field with a FULLTEXT INDEX on it in a MySQL + ** database. Makes a LIKE or CONTAINS (or NOT those) query use the special + ** syntax that hits the FULLTEXT INDEX. + *******************************************************************************/ +public class MySQLFullTextIndexFieldStrategy extends BaseRDBMSActionStrategy +{ + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Integer appendCriterionToWhereClause(QFilterCriteria criterion, StringBuilder clause, String column, List values, QFieldMetaData field) + { + switch(criterion.getOperator()) + { + case LIKE, CONTAINS -> + { + clause.append(" MATCH (").append(column).append(") AGAINST (?) "); + return (1); + } + case NOT_LIKE, NOT_CONTAINS -> + { + clause.append(" NOT MATCH (").append(column).append(") AGAINST (?) "); + return (1); + } + default -> + { + return super.appendCriterionToWhereClause(criterion, clause, column, values, field); + } + } + } +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/RDBMSActionStrategyInterface.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/RDBMSActionStrategyInterface.java new file mode 100644 index 00000000..892d0974 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/RDBMSActionStrategyInterface.java @@ -0,0 +1,103 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.io.Serializable; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface RDBMSActionStrategyInterface +{ + + /*************************************************************************** + * modifies the clause StringBuilder (appending to it) + * returning the number of expected number of params to bind + ***************************************************************************/ + Integer appendCriterionToWhereClause(QFilterCriteria criterion, StringBuilder clause, String column, List values, QFieldMetaData field); + + /*************************************************************************** + * + ***************************************************************************/ + Serializable getFieldValueFromResultSet(QFieldType type, ResultSet resultSet, int i) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + PreparedStatement executeUpdate(Connection connection, String sql, List params) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + void executeBatchUpdate(Connection connection, String updateSQL, List> values) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + List executeInsertForGeneratedIds(Connection connection, String sql, List params, QFieldMetaData primaryKeyField) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + Integer executeUpdateForRowCount(Connection connection, String sql, Object... params) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + void executeStatement(PreparedStatement statement, CharSequence sql, ResultSetProcessor processor, Object... params) throws SQLException, QException; + + + /*************************************************************************** + * + ***************************************************************************/ + Integer getPageSize(AbstractActionInput actionInput); + + + /******************************************************************************* + ** + *******************************************************************************/ + @FunctionalInterface + interface ResultSetProcessor + { + /******************************************************************************* + ** + *******************************************************************************/ + void processResultSet(ResultSet rs) throws SQLException, QException; + } +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java index 08ebe9a9..9e12c9b5 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java @@ -23,10 +23,13 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.sql.Connection; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.module.rdbms.BaseTest; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import org.junit.jupiter.api.AfterEach; @@ -42,9 +45,10 @@ public class RDBMSActionTest extends BaseTest @AfterEach void afterEachRDBMSActionTest() { - QueryManager.resetPageSize(); - QueryManager.resetStatistics(); - QueryManager.setCollectStatistics(false); + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategy(); + actionStrategy.setPageSize(BaseRDBMSActionStrategy.DEFAULT_PAGE_SIZE); + actionStrategy.resetStatistics(); + actionStrategy.setCollectStatistics(false); } @@ -59,6 +63,31 @@ public class RDBMSActionTest extends BaseTest + /*************************************************************************** + * + ***************************************************************************/ + protected static BaseRDBMSActionStrategy getBaseRDBMSActionStrategy() + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME); + BaseRDBMSActionStrategy actionStrategy = (BaseRDBMSActionStrategy) backend.getActionStrategy(); + return actionStrategy; + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected static BaseRDBMSActionStrategy getBaseRDBMSActionStrategyAndActivateCollectingStatistics() + { + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategy(); + actionStrategy.setCollectStatistics(true); + actionStrategy.resetStatistics(); + return actionStrategy; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java index 61e72681..27b17dcd 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java @@ -34,8 +34,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -128,6 +128,11 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest deleteInput.setPrimaryKeys(List.of(1, -1)); DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); assertEquals(1, deleteResult.getDeletedRecordCount(), "Should delete one row"); + + ///////////////////////////////////////////////////////////////////////////////////// + // note - that if we went to the top-level DeleteAction, then it would have pre- // + // checked that the ids existed, and it WOULD give us an error for the -1 row here // + ///////////////////////////////////////////////////////////////////////////////////// assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors (the one not found is just noop)"); } @@ -162,17 +167,15 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// deleteInput.setPrimaryKeys(List.of(1, 2, 3, 4, 5)); - QueryManager.setCollectStatistics(true); - QueryManager.resetStatistics(); - + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); //////////////////////////////////////////////////////////////////////////////////////// // assert that 6 queries ran - the initial delete (which failed), then 5 more deletes // //////////////////////////////////////////////////////////////////////////////////////// - QueryManager.setCollectStatistics(false); - Map queryStats = QueryManager.getStatistics(); - assertEquals(6, queryStats.get(QueryManager.STAT_QUERIES_RAN), "Number of queries ran"); + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(6, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); assertEquals(2, deleteResult.getRecordsWithErrors().size(), "Should get back the 2 records with errors"); assertTrue(deleteResult.getRecordsWithErrors().stream().noneMatch(r -> r.getErrors().isEmpty()), "All we got back should have errors"); @@ -212,17 +215,15 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest //////////////////////////////////////////////////////////////////////// deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(2, 4, 5)))); - QueryManager.setCollectStatistics(true); - QueryManager.resetStatistics(); - + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); ////////////////////////////////// // assert that just 1 query ran // ////////////////////////////////// - QueryManager.setCollectStatistics(false); - Map queryStats = QueryManager.getStatistics(); - assertEquals(1, queryStats.get(QueryManager.STAT_QUERIES_RAN), "Number of queries ran"); + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(1, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); assertEquals(3, deleteResult.getDeletedRecordCount(), "Should get back that 3 were deleted"); runTestSql("SELECT id FROM child_table", (rs -> @@ -259,9 +260,7 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2, 3, 4, 5)))); - QueryManager.setCollectStatistics(true); - QueryManager.resetStatistics(); - + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); /////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -270,9 +269,9 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest // todo - maybe we shouldn't do that 2nd "try to delete 'em all by id"... why would it ever work, // // but the original filter query didn't (other than malformed SQL)? // /////////////////////////////////////////////////////////////////////////////////////////////////////// - QueryManager.setCollectStatistics(false); - Map queryStats = QueryManager.getStatistics(); - assertEquals(8, queryStats.get(QueryManager.STAT_QUERIES_RAN), "Number of queries ran"); + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(8, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); assertEquals(2, deleteResult.getRecordsWithErrors().size(), "Should get back the 2 records with errors"); assertTrue(deleteResult.getRecordsWithErrors().stream().noneMatch(r -> r.getErrors().isEmpty()), "All we got back should have errors"); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java index c7689286..0daa1381 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -32,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -53,6 +54,8 @@ public class RDBMSInsertActionTest extends RDBMSActionTest public void beforeEach() throws Exception { super.primeTestDatabase(); + + getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); } @@ -113,7 +116,7 @@ public class RDBMSInsertActionTest extends RDBMSActionTest @Test public void testInsertMany() throws Exception { - QueryManager.setPageSize(2); + getBaseRDBMSActionStrategy().setPageSize(2); InsertInput insertInput = initInsertRequest(); QRecord record1 = new QRecord().withTableName("person") @@ -137,6 +140,10 @@ public class RDBMSInsertActionTest extends RDBMSActionTest assertEquals(6, insertOutput.getRecords().get(0).getValue("id"), "Should have next id in the row"); assertEquals(7, insertOutput.getRecords().get(1).getValue("id"), "Should have next id in the row"); assertEquals(8, insertOutput.getRecords().get(2).getValue("id"), "Should have next id in the row"); + + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + assertAnInsertedPersonRecord("Jean-Luc", "Picard", 6); assertAnInsertedPersonRecord("William", "Riker", 7); assertAnInsertedPersonRecord("Beverly", "Crusher", 8); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java index 64f4c950..6eb9357f 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java @@ -36,7 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -61,8 +61,7 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest { super.primeTestDatabase(); - QueryManager.setCollectStatistics(true); - QueryManager.resetStatistics(); + getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); } @@ -112,8 +111,8 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest updateInput.setRecords(List.of(record)); UpdateOutput updateResult = new UpdateAction().execute(updateInput); - Map statistics = QueryManager.getStatistics(); - assertEquals(2, statistics.get(QueryManager.STAT_QUERIES_RAN)); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); assertEquals(1, updateResult.getRecords().size(), "Should return 1 row"); assertEquals(2, updateResult.getRecords().get(0).getValue("id"), "Should have id=2 in the row"); @@ -169,9 +168,9 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest UpdateOutput updateResult = new UpdateAction().execute(updateInput); // this test runs one batch and one regular query - Map statistics = QueryManager.getStatistics(); - assertEquals(1, statistics.get(QueryManager.STAT_BATCHES_RAN)); - assertEquals(2, statistics.get(QueryManager.STAT_QUERIES_RAN)); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(1, statistics.get(BaseRDBMSActionStrategy.STAT_BATCHES_RAN)); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); assertEquals(3, updateResult.getRecords().size(), "Should return 3 rows"); assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); @@ -241,8 +240,8 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest updateInput.setRecords(List.of(record1, record2)); UpdateOutput updateResult = new UpdateAction().execute(updateInput); - Map statistics = QueryManager.getStatistics(); - assertEquals(1, statistics.get(QueryManager.STAT_BATCHES_RAN)); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(1, statistics.get(BaseRDBMSActionStrategy.STAT_BATCHES_RAN)); assertEquals(2, updateResult.getRecords().size(), "Should return 2 rows"); assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); @@ -296,8 +295,8 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest updateInput.setRecords(records); UpdateOutput updateResult = new UpdateAction().execute(updateInput); - Map statistics = QueryManager.getStatistics(); - assertEquals(2, statistics.get(QueryManager.STAT_QUERIES_RAN)); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); assertEquals(5, updateResult.getRecords().size(), "Should return 5 rows"); // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java index c588a06a..8657eb68 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java @@ -24,12 +24,17 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -217,6 +222,32 @@ class C3P0PooledConnectionProviderTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueriesForNewConnections() throws Exception + { + String uuid = UUID.randomUUID().toString(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // change the default database backend to use the class under test here - the C3PL connection pool provider // + // and add a new-connection query to it to insert a record with a UUID, that we can then query for. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QInstance qInstance = TestUtils.defineInstance(); + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) qInstance.getBackend(TestUtils.DEFAULT_BACKEND_NAME); + backend.setQueriesForNewConnections(List.of("insert into person (first_name, last_name, email) values ('D', 'K', '" + uuid + "')")); + backend.setConnectionProvider(new QCodeReference(C3P0PooledConnectionProvider.class)); + QContext.init(qInstance, new QSession()); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // we actually get multiple, because c3p0 default config opens multiple connections at once // + ////////////////////////////////////////////////////////////////////////////////////////////// + List records = QueryAction.execute(TestUtils.TABLE_NAME_PERSON, new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.EQUALS, uuid))); + assertThat(records.size()).isGreaterThanOrEqualTo(1); + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionProviderInterfaceTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionProviderInterfaceTest.java new file mode 100644 index 00000000..f71fbcad --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionProviderInterfaceTest.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.jdbc; + + +import java.sql.Connection; +import java.sql.SQLException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for ConnectionProviderInterface + *******************************************************************************/ +class ConnectionProviderInterfaceTest extends BaseTest +{ + + + /*************************************************************************** + * jacoco doesn't like that our interface isn't covered, so... cover it + ***************************************************************************/ + @Test + void dumpDebug() throws SQLException + { + new ConnectionProviderInterface() + { + @Override + public void init(RDBMSBackendMetaData backend) throws QException + { + + } + + + + @Override + public Connection getConnection() throws SQLException + { + return null; + } + }.dumpDebug(); + } +} \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java index 396a8c77..222734fa 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java @@ -422,29 +422,6 @@ class QueryManagerTest extends BaseTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testQueryForSimpleEntity() throws SQLException - { - try(Connection connection = getConnection()) - { - QueryManager.executeUpdate(connection, """ - INSERT INTO test_table - ( int_col, datetime_col, char_col, date_col, time_col ) - VALUES - ( 47, '2022-08-10 19:22:08', 'Q', '2022-08-10', '19:22:08') - """); - SimpleEntity simpleEntity = QueryManager.executeStatementForSimpleEntity(connection, "SELECT * FROM test_table"); - assertNotNull(simpleEntity); - assertEquals(47, simpleEntity.get("INT_COL")); - assertEquals("Q", simpleEntity.get("CHAR_COL")); - } - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategyTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategyTest.java new file mode 100644 index 00000000..95c7784e --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategyTest.java @@ -0,0 +1,161 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.util.GregorianCalendar; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** Unit test for BaseRDBMSActionStrategy + *******************************************************************************/ +class BaseRDBMSActionStrategyTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws SQLException + { + try(Connection connection = getConnection()) + { + QueryManager.executeUpdate(connection, """ + CREATE TABLE test_table + ( + int_col INTEGER, + datetime_col DATETIME, + char_col CHAR(1), + date_col DATE, + time_col TIME, + long_col LONG + ) + """); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() throws SQLException + { + try(Connection connection = getConnection()) + { + QueryManager.executeUpdate(connection, "DROP TABLE test_table"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Connection getConnection() throws SQLException + { + return new ConnectionManager().getConnection(TestUtils.defineBackend()); + } + + + + /******************************************************************************* + ** Test the various overloads that bind params. + ** Note, we're just confirming that these methods don't throw... + *******************************************************************************/ + @Test + void testBindParams() throws SQLException + { + try(Connection connection = getConnection()) + { + long ctMillis = System.currentTimeMillis(); + PreparedStatement ps = connection.prepareStatement("UPDATE test_table SET int_col = ? WHERE int_col > 0"); + + BaseRDBMSActionStrategy strategy = new BaseRDBMSActionStrategy(); + + /////////////////////////////////////////////////////////////////////////////// + // these calls - we just want to assert that they don't throw any exceptions // + /////////////////////////////////////////////////////////////////////////////// + strategy.bindParamObject(ps, 1, (short) 1); + strategy.bindParamObject(ps, 1, (long) 1); + strategy.bindParamObject(ps, 1, true); + strategy.bindParamObject(ps, 1, BigDecimal.ONE); + strategy.bindParamObject(ps, 1, "hello".getBytes(StandardCharsets.UTF_8)); + strategy.bindParamObject(ps, 1, new Timestamp(ctMillis)); + strategy.bindParamObject(ps, 1, new Date(ctMillis)); + strategy.bindParamObject(ps, 1, new GregorianCalendar()); + strategy.bindParamObject(ps, 1, LocalDate.now()); + strategy.bindParamObject(ps, 1, OffsetDateTime.now()); + strategy.bindParamObject(ps, 1, LocalDateTime.now()); + strategy.bindParamObject(ps, 1, AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + assertThrows(SQLException.class, () -> strategy.bindParamObject(ps, 1, new Object())); + + strategy.bindParam(ps, 1, (Integer) null); + strategy.bindParam(ps, 1, (Boolean) null); + strategy.bindParam(ps, 1, (BigDecimal) null); + strategy.bindParam(ps, 1, (byte[]) null); + strategy.bindParam(ps, 1, (Timestamp) null); + strategy.bindParam(ps, 1, (String) null); + strategy.bindParam(ps, 1, (Date) null); + strategy.bindParam(ps, 1, (GregorianCalendar) null); + strategy.bindParam(ps, 1, (LocalDate) null); + strategy.bindParam(ps, 1, (LocalDateTime) null); + + strategy.bindParam(ps, 1, 1); + strategy.bindParam(ps, 1, true); + strategy.bindParam(ps, 1, BigDecimal.ONE); + strategy.bindParam(ps, 1, "hello".getBytes(StandardCharsets.UTF_8)); + strategy.bindParam(ps, 1, new Timestamp(ctMillis)); + strategy.bindParam(ps, 1, "hello"); + strategy.bindParam(ps, 1, new Date(ctMillis)); + strategy.bindParam(ps, 1, new GregorianCalendar()); + strategy.bindParam(ps, 1, LocalDate.now()); + strategy.bindParam(ps, 1, LocalDateTime.now()); + + //////////////////////////////////////////////////////////////////////////////////////////////// + // originally longs were being downgraded to int when binding, so, verify that doesn't happen // + //////////////////////////////////////////////////////////////////////////////////////////////// + } + } + +} \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategyTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategyTest.java new file mode 100644 index 00000000..25c16621 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategyTest.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for MySQLFullTextIndexFieldStrategy + *******************************************************************************/ +class MySQLFullTextIndexFieldStrategyTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + /////////////////////////////////////////////////// + // test an operator that uses the fulltext index // + /////////////////////////////////////////////////// + QFieldMetaData field = new QFieldMetaData("myText", QFieldType.TEXT); + QFilterCriteria criterion = new QFilterCriteria(field.getName(), QCriteriaOperator.LIKE, "hello"); + StringBuilder clause = new StringBuilder(); + Integer expectedNoOfParams = new MySQLFullTextIndexFieldStrategy().appendCriterionToWhereClause(criterion, clause, "my_text", List.of("hello"), field); + assertEquals(1, expectedNoOfParams); + assertEquals(" MATCH (my_text) AGAINST (?) ", clause.toString()); + + //////////////////////////////////////////// + // test a negated fulltext index operator // + //////////////////////////////////////////// + criterion.setOperator(QCriteriaOperator.NOT_CONTAINS); + clause.delete(0, clause.length()); + expectedNoOfParams = new MySQLFullTextIndexFieldStrategy().appendCriterionToWhereClause(criterion, clause, "my_text", List.of("hello"), field); + assertEquals(1, expectedNoOfParams); + assertEquals(" NOT MATCH (my_text) AGAINST (?) ", clause.toString()); + + //////////////////////////////////////////// + // an operator that should defer to super // + //////////////////////////////////////////// + criterion.setOperator(QCriteriaOperator.IS_BLANK); + clause.delete(0, clause.length()); + expectedNoOfParams = new MySQLFullTextIndexFieldStrategy().appendCriterionToWhereClause(criterion, clause, "my_text", List.of("hello"), field); + assertEquals(0, expectedNoOfParams); + assertEquals("my_text IS NULL OR my_text = ''", clause.toString()); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-sqlite/pom.xml b/qqq-backend-module-sqlite/pom.xml new file mode 100644 index 00000000..51c255c6 --- /dev/null +++ b/qqq-backend-module-sqlite/pom.xml @@ -0,0 +1,113 @@ + + + + + 4.0.0 + + qqq-backend-module-sqlite + + + com.kingsrook.qqq + qqq-parent-project + ${revision} + + + + + + + + + + + com.kingsrook.qqq + qqq-backend-module-rdbms + ${revision} + + + + + org.xerial + sqlite-jdbc + 3.47.1.0 + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.assertj + assertj-core + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + *:* + + META-INF/* + + + + + + + ${plugin.shade.phase} + + shade + + + + + + + + diff --git a/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/SQLiteBackendModule.java b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/SQLiteBackendModule.java new file mode 100644 index 00000000..0fb34321 --- /dev/null +++ b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/SQLiteBackendModule.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite; + + +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule; +import com.kingsrook.qqq.backend.module.sqlite.model.metadata.SQLiteBackendMetaData; +import com.kingsrook.qqq.backend.module.sqlite.model.metadata.SQLiteTableBackendDetails; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteBackendModule extends RDBMSBackendModule +{ + private static final QLogger LOG = QLogger.getLogger(SQLiteBackendModule.class); + + private static final String NAME = "sqlite"; + + static + { + QBackendModuleDispatcher.registerBackendModule(new SQLiteBackendModule()); + } + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + public String getBackendType() + { + return NAME; + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (SQLiteBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (SQLiteTableBackendDetails.class); + } + +} diff --git a/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteBackendMetaData.java b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteBackendMetaData.java new file mode 100644 index 00000000..28453737 --- /dev/null +++ b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteBackendMetaData.java @@ -0,0 +1,140 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.model.metadata; + + +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; +import com.kingsrook.qqq.backend.module.sqlite.SQLiteBackendModule; +import com.kingsrook.qqq.backend.module.sqlite.strategy.SQLiteRDBMSActionStrategy; +import org.sqlite.JDBC; + + +/******************************************************************************* + ** Meta-data to provide details of an SQLite backend (e.g., path to the database file) + *******************************************************************************/ +public class SQLiteBackendMetaData extends RDBMSBackendMetaData +{ + private String path; + + // todo - overrides to setters for unsupported fields? + // todo - or - change rdbms connection manager to not require an RDBMSBackendMetaData? + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SQLiteBackendMetaData() + { + super(); + setVendor("sqlite"); + setBackendType(SQLiteBackendModule.class); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String buildConnectionString() + { + return "jdbc:sqlite:" + this.path; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getJdbcDriverClassName() + { + return (JDBC.class.getName()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public SQLiteBackendMetaData withName(String name) + { + setName(name); + return (this); + } + + + + /******************************************************************************* + ** Getter for path + *******************************************************************************/ + public String getPath() + { + return (this.path); + } + + + + /******************************************************************************* + ** Setter for path + *******************************************************************************/ + public void setPath(String path) + { + this.path = path; + } + + + + /******************************************************************************* + ** Fluent setter for path + *******************************************************************************/ + public SQLiteBackendMetaData withPath(String path) + { + this.path = path; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public RDBMSActionStrategyInterface getActionStrategy() + { + if(getActionStrategyField() == null) + { + if(getActionStrategyCodeReference() != null) + { + setActionStrategyField(QCodeLoader.getAdHoc(RDBMSActionStrategyInterface.class, getActionStrategyCodeReference())); + } + else + { + setActionStrategyField(new SQLiteRDBMSActionStrategy()); + } + } + + return (getActionStrategyField()); + } +} diff --git a/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteTableBackendDetails.java b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteTableBackendDetails.java new file mode 100644 index 00000000..7d1d86c6 --- /dev/null +++ b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteTableBackendDetails.java @@ -0,0 +1,66 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.model.metadata; + + +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteTableBackendDetails extends RDBMSTableBackendDetails +{ + private String tableName; + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public SQLiteTableBackendDetails withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + +} diff --git a/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/strategy/SQLiteRDBMSActionStrategy.java b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/strategy/SQLiteRDBMSActionStrategy.java new file mode 100644 index 00000000..85d0c528 --- /dev/null +++ b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/strategy/SQLiteRDBMSActionStrategy.java @@ -0,0 +1,154 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.strategy; + + +import java.io.Serializable; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; + + +/******************************************************************************* + ** SQLite specialization of the default RDBMS/JDBC action strategy + *******************************************************************************/ +public class SQLiteRDBMSActionStrategy extends BaseRDBMSActionStrategy +{ + + /*************************************************************************** + ** deal with sqlite not having temporal types... so temporal values + ** i guess are stored as strings, as that's how they come back to us - so + ** the JDBC methods fail trying to getDate or whatever from them - but + ** getting the values as strings, they parse nicely, so do that. + ***************************************************************************/ + @Override + public Serializable getFieldValueFromResultSet(QFieldType type, ResultSet resultSet, int i) throws SQLException + { + return switch(type) + { + case DATE -> + { + try + { + yield parseString(s -> LocalDate.parse(s), resultSet, i); + } + catch(Exception e) + { + ///////////////////////////////////////////////////////////////////////////////// + // handle the case of, the value we got back is actually a date-time -- so -- // + // let's parse it as such, and then map into a LocalDate in the session zoneId // + ///////////////////////////////////////////////////////////////////////////////// + Instant instant = (Instant) parseString(s -> Instant.parse(s), resultSet, i); + if(instant == null) + { + yield null; + } + ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId(); + yield instant.atZone(zoneId).toLocalDate(); + } + } + case TIME -> parseString(s -> LocalTime.parse(s), resultSet, i); + case DATE_TIME -> parseString(s -> Instant.parse(s), resultSet, i); + default -> super.getFieldValueFromResultSet(type, resultSet, i); + }; + } + + + + /*************************************************************************** + ** helper method for getFieldValueFromResultSet + ***************************************************************************/ + private Serializable parseString(Function parser, ResultSet resultSet, int i) throws SQLException + { + String valueString = QueryManager.getString(resultSet, i); + if(valueString == null) + { + return (null); + } + else + { + return parser.apply(valueString); + } + } + + + + /*************************************************************************** + * bind temporal types as strings (see above comment re: sqlite temporal types) + ***************************************************************************/ + @Override + protected int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException + { + if(value instanceof Instant || value instanceof LocalTime || value instanceof LocalDate) + { + bindParam(statement, index, value.toString()); + return 1; + } + else + { + return super.bindParamObject(statement, index, value); + } + } + + + + /*************************************************************************** + ** per discussion (and rejected PR mentioned) on https://github.com/prrvchr/sqlite-jdbc + ** sqlite jdbc by default will only return the latest generated serial. but we can get + ** them all by appending this "RETURNING id" to the query, and then calling execute() + ** (instead of executeUpdate()) and getResultSet (instead of getGeneratedKeys()) + ***************************************************************************/ + @Override + public List executeInsertForGeneratedIds(Connection connection, String sql, List params, QFieldMetaData primaryKeyField) throws SQLException + { + sql = sql + " RETURNING " + getColumnName(primaryKeyField); + + try(PreparedStatement statement = connection.prepareStatement(sql)) + { + bindParams(params.toArray(), statement); + incrementStatistic(STAT_QUERIES_RAN); + statement.execute(); + + ResultSet generatedKeys = statement.getResultSet(); + List rs = new ArrayList<>(); + while(generatedKeys.next()) + { + rs.add(getFieldValueFromResultSet(primaryKeyField.getType(), generatedKeys, 1)); + } + return (rs); + } + } + +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/BaseTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/BaseTest.java new file mode 100644 index 00000000..82a3556a --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/BaseTest.java @@ -0,0 +1,124 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite; + + +import java.sql.Connection; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BaseTest +{ + private static final QLogger LOG = QLogger.getLogger(BaseTest.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void baseBeforeEach() throws Exception + { + QContext.init(TestUtils.defineInstance(), new QSession()); + TestUtils.primeTestDatabase("prime-test-database.sql"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void baseAfterEach() + { + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategy(); + actionStrategy.setPageSize(BaseRDBMSActionStrategy.DEFAULT_PAGE_SIZE); + actionStrategy.resetStatistics(); + actionStrategy.setCollectStatistics(false); + + QContext.clear(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected static BaseRDBMSActionStrategy getBaseRDBMSActionStrategy() + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME); + BaseRDBMSActionStrategy actionStrategy = (BaseRDBMSActionStrategy) backend.getActionStrategy(); + return actionStrategy; + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected static BaseRDBMSActionStrategy getBaseRDBMSActionStrategyAndActivateCollectingStatistics() + { + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategy(); + actionStrategy.setCollectStatistics(true); + actionStrategy.resetStatistics(); + return actionStrategy; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected static void reInitInstanceInContext(QInstance qInstance) + { + if(qInstance.equals(QContext.getQInstance())) + { + LOG.warn("Unexpected condition - the same qInstance that is already in the QContext was passed into reInit. You probably want a new QInstance object instance."); + } + QContext.init(qInstance, new QSession()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void runTestSql(String sql, QueryManager.ResultSetProcessor resultSetProcessor) throws Exception + { + ConnectionManager connectionManager = new ConnectionManager(); + Connection connection = connectionManager.getConnection(TestUtils.defineBackend()); + QueryManager.executeStatement(connection, sql, resultSetProcessor); + connection.close(); + } +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/TestUtils.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/TestUtils.java new file mode 100644 index 00000000..7fb9feb4 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/TestUtils.java @@ -0,0 +1,460 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite; + + +import java.io.File; +import java.io.InputStream; +import java.sql.Connection; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.C3P0PooledConnectionProvider; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.sqlite.model.metadata.SQLiteBackendMetaData; +import com.kingsrook.qqq.backend.module.sqlite.model.metadata.SQLiteTableBackendDetails; +import org.apache.commons.io.IOUtils; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestUtils +{ + public static final String DEFAULT_BACKEND_NAME = "default"; + public static final String MEMORY_BACKEND_NAME = "memory"; + + public static final String TABLE_NAME_PERSON = "personTable"; + public static final String TABLE_NAME_PERSONAL_ID_CARD = "personalIdCard"; + public static final String TABLE_NAME_STORE = "store"; + public static final String TABLE_NAME_ORDER = "order"; + public static final String TABLE_NAME_ORDER_INSTRUCTIONS = "orderInstructions"; + public static final String TABLE_NAME_ITEM = "item"; + public static final String TABLE_NAME_ORDER_LINE = "orderLine"; + public static final String TABLE_NAME_LINE_ITEM_EXTRINSIC = "orderLineExtrinsic"; + public static final String TABLE_NAME_WAREHOUSE = "warehouse"; + public static final String TABLE_NAME_WAREHOUSE_STORE_INT = "warehouseStoreInt"; + + public static final String SECURITY_KEY_STORE_ALL_ACCESS = "storeAllAccess"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static void primeTestDatabase(String sqlFileName) throws Exception + { + SQLiteBackendMetaData backend = TestUtils.defineBackend(); + + File file = new File(backend.getPath()); + file.getParentFile().mkdirs(); + + try(Connection connection = ConnectionManager.getConnection(backend)) + { + InputStream primeTestDatabaseSqlStream = SQLiteBackendModule.class.getResourceAsStream("/" + sqlFileName); + assertNotNull(primeTestDatabaseSqlStream); + List lines = (List) IOUtils.readLines(primeTestDatabaseSqlStream); + lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); + String joinedSQL = String.join("\n", lines); + for(String sql : joinedSQL.split(";")) + { + if(sql.matches("(?s).*[a-zA-Z0-9_].*")) + { + QueryManager.executeUpdate(connection, sql); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QInstance defineInstance() + { + QInstance qInstance = new QInstance(); + qInstance.addBackend(defineBackend()); + qInstance.addBackend(defineMemoryBackend()); + qInstance.addTable(defineTablePerson()); + qInstance.addPossibleValueSource(definePvsPerson()); + qInstance.addTable(defineTablePersonalIdCard()); + qInstance.addJoin(defineJoinPersonAndPersonalIdCard()); + addOmsTablesAndJoins(qInstance); + qInstance.setAuthentication(defineAuthentication()); + return (qInstance); + } + + + + /******************************************************************************* + ** Define the in-memory backend used in standard tests + *******************************************************************************/ + public static QBackendMetaData defineMemoryBackend() + { + return new QBackendMetaData() + .withName(MEMORY_BACKEND_NAME) + .withBackendType(MemoryBackendModule.class); + } + + + + /******************************************************************************* + ** Define the authentication used in standard tests - using 'mock' type. + ** + *******************************************************************************/ + public static QAuthenticationMetaData defineAuthentication() + { + return new QAuthenticationMetaData() + .withName("mock") + .withType(QAuthenticationType.MOCK); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static SQLiteBackendMetaData defineBackend() + { + SQLiteBackendMetaData sqLiteBackendMetaData = new SQLiteBackendMetaData() + .withName(DEFAULT_BACKEND_NAME) + .withPath("/tmp/sqlite/test.db"); + + sqLiteBackendMetaData.setQueriesForNewConnections(List.of( + "PRAGMA foreign_keys = ON" + )); + + sqLiteBackendMetaData.setConnectionProvider(new QCodeReference(C3P0PooledConnectionProvider.class)); + + return sqLiteBackendMetaData; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineTablePerson() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON) + .withLabel("Person") + .withRecordLabelFormat("%s %s") + .withRecordLabelFields("firstName", "lastName") + .withBackendName(DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + .withField(new QFieldMetaData("email", QFieldType.STRING).withBackendName("email")) + .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN).withBackendName("is_employed")) + .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary")) + .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked")) + .withField(new QFieldMetaData("homeTown", QFieldType.STRING).withBackendName("home_town")) + .withField(new QFieldMetaData("startTime", QFieldType.TIME).withBackendName("start_time")) + .withBackendDetails(new SQLiteTableBackendDetails() + .withTableName("person")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValueSource definePvsPerson() + { + return (new QPossibleValueSource() + .withName(TABLE_NAME_PERSON) + .withType(QPossibleValueSourceType.TABLE) + .withTableName(TABLE_NAME_PERSON) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)); + } + + + + /******************************************************************************* + ** Define a 1:1 table with Person. + ** + *******************************************************************************/ + private static QTableMetaData defineTablePersonalIdCard() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSONAL_ID_CARD) + .withLabel("Personal Id Card") + .withBackendName(DEFAULT_BACKEND_NAME) + .withBackendDetails(new RDBMSTableBackendDetails() + .withTableName("personal_id_card")) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("personId", QFieldType.INTEGER).withBackendName("person_id")) + .withField(new QFieldMetaData("idNumber", QFieldType.STRING).withBackendName("id_number")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QJoinMetaData defineJoinPersonAndPersonalIdCard() + { + return new QJoinMetaData() + .withLeftTable(TABLE_NAME_PERSON) + .withRightTable(TABLE_NAME_PERSONAL_ID_CARD) + .withInferredName() + .withType(JoinType.ONE_TO_ONE) + .withJoinOn(new JoinOn("id", "personId")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void addOmsTablesAndJoins(QInstance qInstance) + { + qInstance.addTable(defineBaseTable(TABLE_NAME_STORE, "store") + .withRecordLabelFormat("%s") + .withRecordLabelFields("name") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("id")) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER, "order") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) + .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem"))) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")).withLabel("Current Order Instructions")) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) + .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("currentOrderInstructionsId", QFieldType.INTEGER).withBackendName("current_order_instructions_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_INSTRUCTIONS, "order_instructions") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeId") + .withJoinNameChain(List.of("orderInstructionsJoinOrder"))) + .withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id")) + .withField(new QFieldMetaData("instructions", QFieldType.STRING)) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderInstructionsJoinOrder"))) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderLineJoinItem", "orderJoinOrderLine"))) + .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("description", QFieldType.STRING)) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_LINE, "order_line") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeId") + .withJoinNameChain(List.of("orderJoinOrderLine"))) + .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_LINE_ITEM_EXTRINSIC).withJoinName("orderLineJoinLineItemExtrinsic")) + .withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id")) + .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) + .withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_LINE_ITEM_EXTRINSIC, "line_item_extrinsic") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeId") + .withJoinNameChain(List.of("orderJoinOrderLine", "orderLineJoinLineItemExtrinsic"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("orderLineId", QFieldType.INTEGER).withBackendName("order_line_id")) + .withField(new QFieldMetaData("key", QFieldType.STRING)) + .withField(new QFieldMetaData("value", QFieldType.STRING)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE_STORE_INT, "warehouse_store_int") + .withField(new QFieldMetaData("warehouseId", QFieldType.INTEGER).withBackendName("warehouse_id")) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id")) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE, "warehouse") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName(TABLE_NAME_WAREHOUSE_STORE_INT + ".storeId") + .withJoinNameChain(List.of(QJoinMetaData.makeInferredJoinName(TestUtils.TABLE_NAME_WAREHOUSE, TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT))) + ) + .withField(new QFieldMetaData("name", QFieldType.STRING).withBackendName("name")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withType(JoinType.ONE_TO_MANY) + .withLeftTable(TestUtils.TABLE_NAME_WAREHOUSE) + .withRightTable(TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT) + .withInferredName() + .withJoinOn(new JoinOn("id", "warehouseId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinStore") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_STORE) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("storeId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinBillToPerson") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_PERSON) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("billToPersonId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinShipToPerson") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_PERSON) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("shipToPersonId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("itemJoinStore") + .withLeftTable(TABLE_NAME_ITEM) + .withRightTable(TABLE_NAME_STORE) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("storeId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinOrderLine") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_ORDER_LINE) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "orderId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderLineJoinItem") + .withLeftTable(TABLE_NAME_ORDER_LINE) + .withRightTable(TABLE_NAME_ITEM) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("sku", "sku")) + .withJoinOn(new JoinOn("storeId", "storeId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderLineJoinLineItemExtrinsic") + .withLeftTable(TABLE_NAME_ORDER_LINE) + .withRightTable(TABLE_NAME_LINE_ITEM_EXTRINSIC) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "orderLineId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinCurrentOrderInstructions") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS) + .withType(JoinType.ONE_TO_ONE) + .withJoinOn(new JoinOn("currentOrderInstructionsId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderInstructionsJoinOrder") + .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS) + .withLeftTable(TABLE_NAME_ORDER) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("id", "orderId")) + ); + + qInstance.addPossibleValueSource(new QPossibleValueSource() + .withName("store") + .withType(QPossibleValueSourceType.TABLE) + .withTableName(TABLE_NAME_STORE) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY) + ); + + qInstance.addSecurityKeyType(new QSecurityKeyType() + .withName(TABLE_NAME_STORE) + .withAllAccessKeyName(SECURITY_KEY_STORE_ALL_ACCESS) + .withPossibleValueSourceName(TABLE_NAME_STORE)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QTableMetaData defineBaseTable(String tableName, String backendTableName) + { + return new QTableMetaData() + .withName(tableName) + .withBackendName(DEFAULT_BACKEND_NAME) + .withBackendDetails(new RDBMSTableBackendDetails().withTableName(backendTableName)) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List queryTable(String tableName) throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(tableName); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return (queryOutput.getRecords()); + } +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteCountActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteCountActionTest.java new file mode 100644 index 00000000..64ae6390 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteCountActionTest.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteCountActionTest extends BaseTest +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUnfilteredCount() throws QException + { + CountInput countInput = initCountRequest(); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(5, countOutput.getCount(), "Unfiltered query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEqualsQueryCount() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + CountInput countInput = initCountRequest(); + countInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.EQUALS) + .withValues(List.of(email))) + ); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(1, countOutput.getCount(), "Expected # of rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + CountInput countInput = initCountRequest(); + countInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(email))) + ); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(4, countOutput.getCount(), "Expected # of rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private CountInput initCountRequest() + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.defineTablePerson().getName()); + return countInput; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneInnerJoinWithoutWhere() throws QException + { + CountInput countInput = initCountRequest(); + countInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD)); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(3, countOutput.getCount(), "Join count should find 3 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneLeftJoinWithoutWhere() throws QException + { + CountInput countInput = initCountRequest(); + countInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withType(QueryJoin.Type.LEFT)); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(5, countOutput.getCount(), "Left Join count should find 5 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneRightJoinWithoutWhere() throws QException + { + CountInput countInput = initCountRequest(); + countInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withType(QueryJoin.Type.RIGHT)); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(6, countOutput.getCount(), "Right Join count should find 6 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneInnerJoinWithWhere() throws QException + { + CountInput countInput = initCountRequest(); + countInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true)); + countInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber", QCriteriaOperator.STARTS_WITH, "1980"))); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(2, countOutput.getCount(), "Right Join count should find 2 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurity() throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + QContext.setQSession(new QSession()); + assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(0); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(8); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(5); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE); + + assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(4); + } + +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteDeleteActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteDeleteActionTest.java new file mode 100644 index 00000000..d5734f85 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteDeleteActionTest.java @@ -0,0 +1,322 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSDeleteAction; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteDeleteActionTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteAll() throws Exception + { + DeleteInput deleteInput = initStandardPersonDeleteRequest(); + deleteInput.setPrimaryKeys(List.of(1, 2, 3, 4, 5)); + DeleteOutput deleteResult = new DeleteAction().execute(deleteInput); + assertEquals(5, deleteResult.getDeletedRecordCount(), "Unfiltered delete should return all rows"); + assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors"); + runTestSql("SELECT id FROM person", (rs -> assertFalse(rs.next()))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteOne() throws Exception + { + DeleteInput deleteInput = initStandardPersonDeleteRequest(); + deleteInput.setPrimaryKeys(List.of(1)); + DeleteOutput deleteResult = new DeleteAction().execute(deleteInput); + assertEquals(1, deleteResult.getDeletedRecordCount(), "Should delete one row"); + assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors"); + runTestSql("SELECT id FROM person WHERE id = 1", (rs -> assertFalse(rs.next()))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteSome() throws Exception + { + DeleteInput deleteInput = initStandardPersonDeleteRequest(); + deleteInput.setPrimaryKeys(List.of(1, 3, 5)); + DeleteOutput deleteResult = new DeleteAction().execute(deleteInput); + assertEquals(3, deleteResult.getDeletedRecordCount(), "Should delete one row"); + assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors"); + runTestSql("SELECT id FROM person", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertTrue(rs.getInt(1) == 2 || rs.getInt(1) == 4); + } + assertEquals(2, rowsFound); + })); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDeleteSomeIdsThatExistAndSomeThatDoNot() throws Exception + { + DeleteInput deleteInput = initStandardPersonDeleteRequest(); + deleteInput.setPrimaryKeys(List.of(1, -1)); + DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); + assertEquals(1, deleteResult.getDeletedRecordCount(), "Should delete one row"); + + ///////////////////////////////////////////////////////////////////////////////////// + // note - that if we went to the top-level DeleteAction, then it would have pre- // + // checked that the ids existed, and it WOULD give us an error for the -1 row here // + ///////////////////////////////////////////////////////////////////////////////////// + assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors (the one not found is just noop)"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private DeleteInput initStandardPersonDeleteRequest() + { + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(TestUtils.defineTablePerson().getName()); + return deleteInput; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteWhereForeignKeyBlocksSome() throws Exception + { + ////////////////////////////////////////////////////////////////// + // load the parent-child tables, with foreign keys and instance // + ////////////////////////////////////////////////////////////////// + TestUtils.primeTestDatabase("prime-test-database-parent-child-tables.sql"); + DeleteInput deleteInput = initChildTableInstanceAndDeleteRequest(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // try to delete all of the child records - 2 should fail, because they are referenced by parent_table.child_id // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + deleteInput.setPrimaryKeys(List.of(1, 2, 3, 4, 5)); + + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); + DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); + + //////////////////////////////////////////////////////////////////////////////////////// + // assert that 6 queries ran - the initial delete (which failed), then 5 more deletes // + //////////////////////////////////////////////////////////////////////////////////////// + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(6, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); + + assertEquals(2, deleteResult.getRecordsWithErrors().size(), "Should get back the 2 records with errors"); + assertTrue(deleteResult.getRecordsWithErrors().stream().noneMatch(r -> r.getErrors().isEmpty()), "All we got back should have errors"); + assertEquals(3, deleteResult.getDeletedRecordCount(), "Should get back that 3 were deleted"); + + runTestSql("SELECT id FROM child_table", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + /////////////////////////////////////////// + // child_table rows 1 & 3 should survive // + /////////////////////////////////////////// + assertTrue(rs.getInt(1) == 1 || rs.getInt(1) == 3); + } + assertEquals(2, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteByFilterThatJustWorks() throws Exception + { + ////////////////////////////////////////////////////////////////// + // load the parent-child tables, with foreign keys and instance // + ////////////////////////////////////////////////////////////////// + TestUtils.primeTestDatabase("prime-test-database-parent-child-tables.sql"); + DeleteInput deleteInput = initChildTableInstanceAndDeleteRequest(); + + //////////////////////////////////////////////////////////////////////// + // try to delete the records without a foreign key that'll block them // + //////////////////////////////////////////////////////////////////////// + deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(2, 4, 5)))); + + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); + DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); + + ////////////////////////////////// + // assert that just 1 query ran // + ////////////////////////////////// + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(1, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); + assertEquals(3, deleteResult.getDeletedRecordCount(), "Should get back that 3 were deleted"); + + runTestSql("SELECT id FROM child_table", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + /////////////////////////////////////////// + // child_table rows 1 & 3 should survive // + /////////////////////////////////////////// + assertTrue(rs.getInt(1) == 1 || rs.getInt(1) == 3); + } + assertEquals(2, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteByFilterWhereForeignKeyBlocksSome() throws Exception + { + ////////////////////////////////////////////////////////////////// + // load the parent-child tables, with foreign keys and instance // + ////////////////////////////////////////////////////////////////// + TestUtils.primeTestDatabase("prime-test-database-parent-child-tables.sql"); + DeleteInput deleteInput = initChildTableInstanceAndDeleteRequest(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // try to delete all of the child records - 2 should fail, because they are referenced by parent_table.child_id // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2, 3, 4, 5)))); + + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); + DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that 8 queries ran - the initial delete (which failed), then 1 to look up the ids // + // from that query, another to try to delete all those ids (also fails), and finally 5 deletes by id // + // todo - maybe we shouldn't do that 2nd "try to delete 'em all by id"... why would it ever work, // + // but the original filter query didn't (other than malformed SQL)? // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(8, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); + + assertEquals(2, deleteResult.getRecordsWithErrors().size(), "Should get back the 2 records with errors"); + assertTrue(deleteResult.getRecordsWithErrors().stream().noneMatch(r -> r.getErrors().isEmpty()), "All we got back should have errors"); + assertEquals(3, deleteResult.getDeletedRecordCount(), "Should get back that 3 were deleted"); + + runTestSql("SELECT id FROM child_table", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + /////////////////////////////////////////// + // child_table rows 1 & 3 should survive // + /////////////////////////////////////////// + assertTrue(rs.getInt(1) == 1 || rs.getInt(1) == 3); + } + assertEquals(2, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private DeleteInput initChildTableInstanceAndDeleteRequest() + { + QInstance qInstance = TestUtils.defineInstance(); + + String childTableName = "childTable"; + qInstance.addTable(new QTableMetaData() + .withName(childTableName) + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + .withBackendDetails(new RDBMSTableBackendDetails() + .withTableName("child_table"))); + + qInstance.addTable(new QTableMetaData() + .withName("parentTable") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + .withField(new QFieldMetaData("childId", QFieldType.INTEGER).withBackendName("child_id")) + .withBackendDetails(new RDBMSTableBackendDetails() + .withTableName("parent_table"))); + + reInitInstanceInContext(qInstance); + + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(childTableName); + + return deleteInput; + } +} \ No newline at end of file diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteInsertActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteInsertActionTest.java new file mode 100644 index 00000000..760b4ad5 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteInsertActionTest.java @@ -0,0 +1,219 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteInsertActionTest extends BaseTest +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertNullList() throws QException + { + InsertInput insertInput = initInsertRequest(); + insertInput.setRecords(null); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(0, insertOutput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertEmptyList() throws QException + { + InsertInput insertInput = initInsertRequest(); + insertInput.setRecords(Collections.emptyList()); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(0, insertOutput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertOne() throws Exception + { + InsertInput insertInput = initInsertRequest(); + QRecord record = new QRecord().withTableName("person") + .withValue("firstName", "James") + .withValue("lastName", "Kirk") + .withValue("email", "jamestk@starfleet.net") + .withValue("birthDate", "2210-05-20"); + insertInput.setRecords(List.of(record)); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(1, insertOutput.getRecords().size(), "Should return 1 row"); + assertNotNull(insertOutput.getRecords().get(0).getValue("id"), "Should have an id in the row"); + // todo - add errors to QRecord? assertTrue(insertResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + assertAnInsertedPersonRecord("James", "Kirk", 6); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertMany() throws Exception + { + getBaseRDBMSActionStrategyAndActivateCollectingStatistics() + .setPageSize(2); + + InsertInput insertInput = initInsertRequest(); + QRecord record1 = new QRecord().withTableName("person") + .withValue("firstName", "Jean-Luc") + .withValue("lastName", "Picard") + .withValue("email", "jl@starfleet.net") + .withValue("birthDate", "2310-05-20"); + QRecord record2 = new QRecord().withTableName("person") + .withValue("firstName", "William") + .withValue("lastName", "Riker") + .withValue("email", "notthomas@starfleet.net") + .withValue("birthDate", "2320-05-20"); + QRecord record3 = new QRecord().withTableName("person") + .withValue("firstName", "Beverly") + .withValue("lastName", "Crusher") + .withValue("email", "doctor@starfleet.net") + .withValue("birthDate", "2320-06-26"); + insertInput.setRecords(List.of(record1, record2, record3)); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(3, insertOutput.getRecords().size(), "Should return right # of rows"); + assertEquals(6, insertOutput.getRecords().get(0).getValue("id"), "Should have next id in the row"); + assertEquals(7, insertOutput.getRecords().get(1).getValue("id"), "Should have next id in the row"); + assertEquals(8, insertOutput.getRecords().get(2).getValue("id"), "Should have next id in the row"); + + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + + assertAnInsertedPersonRecord("Jean-Luc", "Picard", 6); + assertAnInsertedPersonRecord("William", "Riker", 7); + assertAnInsertedPersonRecord("Beverly", "Crusher", 8); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInsertAssociations() throws QException + { + QContext.getQSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1); + + int originalNoOfOrderLineExtrinsics = TestUtils.queryTable(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).size(); + int originalNoOfOrderLines = TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER_LINE).size(); + int originalNoOfOrders = TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER).size(); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_ORDER); + insertInput.setRecords(List.of( + new QRecord().withValue("storeId", 1).withValue("billToPersonId", 100).withValue("shipToPersonId", 200) + + .withAssociatedRecord("orderLine", new QRecord().withValue("storeId", 1).withValue("sku", "BASIC1").withValue("quantity", 1) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-1.1").withValue("value", "LINE-VAL-1"))) + + .withAssociatedRecord("orderLine", new QRecord().withValue("storeId", 1).withValue("sku", "BASIC2").withValue("quantity", 2) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-2.1").withValue("value", "LINE-VAL-2")) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-2.2").withValue("value", "LINE-VAL-3"))) + )); + new InsertAction().execute(insertInput); + + List orders = TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER); + assertEquals(originalNoOfOrders + 1, orders.size()); + assertTrue(orders.stream().anyMatch(r -> Objects.equals(r.getValue("billToPersonId"), 100) && Objects.equals(r.getValue("shipToPersonId"), 200))); + + List orderLines = TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER_LINE); + assertEquals(originalNoOfOrderLines + 2, orderLines.size()); + assertTrue(orderLines.stream().anyMatch(r -> Objects.equals(r.getValue("sku"), "BASIC1") && Objects.equals(r.getValue("quantity"), 1))); + assertTrue(orderLines.stream().anyMatch(r -> Objects.equals(r.getValue("sku"), "BASIC2") && Objects.equals(r.getValue("quantity"), 2))); + + List lineItemExtrinsics = TestUtils.queryTable(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC); + assertEquals(originalNoOfOrderLineExtrinsics + 3, lineItemExtrinsics.size()); + assertTrue(lineItemExtrinsics.stream().anyMatch(r -> Objects.equals(r.getValue("key"), "LINE-EXT-1.1") && Objects.equals(r.getValue("value"), "LINE-VAL-1"))); + assertTrue(lineItemExtrinsics.stream().anyMatch(r -> Objects.equals(r.getValue("key"), "LINE-EXT-2.1") && Objects.equals(r.getValue("value"), "LINE-VAL-2"))); + assertTrue(lineItemExtrinsics.stream().anyMatch(r -> Objects.equals(r.getValue("key"), "LINE-EXT-2.2") && Objects.equals(r.getValue("value"), "LINE-VAL-3"))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertAnInsertedPersonRecord(String firstName, String lastName, Integer id) throws Exception + { + runTestSql("SELECT * FROM person WHERE last_name = '" + lastName + "'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(id, rs.getInt("id")); + assertEquals(firstName, rs.getString("first_name")); + assertNotNull(rs.getString("create_date")); + assertNotNull(rs.getString("modify_date")); + } + assertEquals(1, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private InsertInput initInsertRequest() + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + return insertInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteQueryActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteQueryActionTest.java new file mode 100644 index 00000000..0108bb8e --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteQueryActionTest.java @@ -0,0 +1,1136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.io.Serializable; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Predicate; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.ThisOrLastPeriod; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import com.kingsrook.qqq.backend.module.rdbms.actions.AbstractRDBMSAction; +import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSQueryAction; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteQueryActionTest extends BaseTest +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + AbstractRDBMSAction.setLogSQL(false); + QContext.getQSession().removeValue(QSession.VALUE_KEY_USER_TIMEZONE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUnfilteredQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testTrueQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.TRUE))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "'TRUE' query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFalseQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.FALSE))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "'FALSE' query should find no rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.EQUALS) + .withValues(List.of(email))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertEquals(email, queryOutput.getRecords().get(0).getValueString("email"), "Should find expected email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(email))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsOrIsNullQuery() throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // 5 rows, 1 has a null salary, 1 has 1,000,000. // + // first confirm that query for != returns 3 (the null does NOT come back) // + // then, confirm that != or is null gives the (more humanly expected) 4. // + ///////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("annualSalary") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(1_000_000)))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows"); + + queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("annualSalary") + .withOperator(QCriteriaOperator.NOT_EQUALS_OR_IS_NULL) + .withValues(List.of(1_000_000)))); + queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.IN) + .withValues(List.of(2, 4))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(4)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotInQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.NOT_IN) + .withValues(List.of(2, 3, 4))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testStartsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.STARTS_WITH) + .withValues(List.of("darin"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testContains() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.CONTAINS) + .withValues(List.of("kelkhoff"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLike() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.LIKE) + .withValues(List.of("%kelk%"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotLike() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_LIKE) + .withValues(List.of("%kelk%"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEndsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.ENDS_WITH) + .withValues(List.of("gmail.com"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotStartsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_STARTS_WITH) + .withValues(List.of("darin"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotContains() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_CONTAINS) + .withValues(List.of("kelkhoff"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEndsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_ENDS_WITH) + .withValues(List.of("gmail.com"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLessThanQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.LESS_THAN) + .withValues(List.of(3))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLessThanOrEqualsQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS) + .withValues(List.of(2))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGreaterThanQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.GREATER_THAN) + .withValues(List.of(3))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGreaterThanOrEqualsQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.GREATER_THAN_OR_EQUALS) + .withValues(List.of(4))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testIsBlankQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("birthDate") + .withOperator(QCriteriaOperator.IS_BLANK) + )); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testIsNotBlankQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("firstName") + .withOperator(QCriteriaOperator.IS_NOT_BLANK) + )); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testBetweenQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.BETWEEN) + .withValues(List.of(2, 4)) + )); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(3) || r.getValueInteger("id").equals(4)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotBetweenQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.NOT_BETWEEN) + .withValues(List.of(2, 4)) + )); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFilterExpressions() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + insertInput.setRecords(List.of( + new QRecord().withValue("email", "-").withValue("firstName", "past").withValue("lastName", "ExpressionTest").withValue("birthDate", Instant.now().minus(3, ChronoUnit.DAYS)), + new QRecord().withValue("email", "-").withValue("firstName", "future").withValue("lastName", "ExpressionTest").withValue("birthDate", Instant.now().plus(3, ChronoUnit.DAYS)) + )); + new InsertAction().execute(insertInput); + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(new Now())))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + } + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(NowWithOffset.plus(2, ChronoUnit.DAYS))))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + } + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withValues(List.of(NowWithOffset.minus(5, ChronoUnit.DAYS))))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row"); + } + } + + + + /******************************************************************************* + ** Adding additional test conditions, specifically for DATE-precision + *******************************************************************************/ + @ParameterizedTest() + @ValueSource(strings = { "UTC", "US/Eastern", "UTC+12" }) + void testMoreFilterExpressions(String userTimezone) throws QException + { + QContext.getQSession().setValue(QSession.VALUE_KEY_USER_TIMEZONE, userTimezone); + + LocalDate today = Instant.now().atZone(ZoneId.of(userTimezone)).toLocalDate(); + LocalDate yesterday = today.minusDays(1); + LocalDate tomorrow = today.plusDays(1); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON).withRecords(List.of( + new QRecord().withValue("email", "-").withValue("firstName", "yesterday").withValue("lastName", "ExpressionTest").withValue("birthDate", yesterday), + new QRecord().withValue("email", "-").withValue("firstName", "today").withValue("lastName", "ExpressionTest").withValue("birthDate", today), + new QRecord().withValue("email", "-").withValue("firstName", "tomorrow").withValue("lastName", "ExpressionTest").withValue("birthDate", tomorrow)) + )); + + UnsafeFunction, List, QException> testFunction = (filterConsumer) -> + { + QQueryFilter filter = new QQueryFilter().withCriteria("lastName", QCriteriaOperator.EQUALS, "ExpressionTest"); + filter.withOrderBy(new QFilterOrderBy("birthDate")); + filterConsumer.accept(filter); + + return QueryAction.execute(TestUtils.TABLE_NAME_PERSON, filter); + }; + + assertOneRecordWithFirstName("today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, new Now())))); + assertOneRecordWithFirstName("tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, new Now())))); + assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, new Now())))); + assertTwoRecordsWithFirstNames("yesterday", "today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN_OR_EQUALS, new Now())))); + assertTwoRecordsWithFirstNames("today", "tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, new Now())))); + + assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.minus(1, ChronoUnit.DAYS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN_OR_EQUALS, NowWithOffset.plus(1, ChronoUnit.DAYS))))); + assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, NowWithOffset.minus(1, ChronoUnit.DAYS))))); + assertOneRecordWithFirstName("tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, NowWithOffset.plus(1, ChronoUnit.DAYS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.WEEKS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.MONTHS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.YEARS))))); + + assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.HOURS))))) + .hasRootCauseMessage("Unsupported unit: Hours"); + + assertOneRecordWithFirstName("today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, ThisOrLastPeriod.this_(ChronoUnit.DAYS))))); + assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, ThisOrLastPeriod.last(ChronoUnit.DAYS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.WEEKS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.MONTHS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.YEARS))))); + assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.WEEKS))))); + assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.MONTHS))))); + assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.YEARS))))); + + assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.this_(ChronoUnit.HOURS))))) + .hasRootCauseMessage("Unsupported unit: Hours"); + assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.MINUTES))))) + .hasRootCauseMessage("Unsupported unit: Minutes"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertNoOfRecords(Integer expectedSize, List actualRecords) + { + assertEquals(expectedSize, actualRecords.size()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertOneRecordWithFirstName(String expectedFirstName, List actualRecords) + { + assertEquals(1, actualRecords.size()); + assertEquals(expectedFirstName, actualRecords.get(0).getValueString("firstName")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertTwoRecordsWithFirstNames(String expectedFirstName0, String expectedFirstName1, List actualRecords) + { + assertEquals(2, actualRecords.size()); + assertEquals(expectedFirstName0, actualRecords.get(0).getValueString("firstName")); + assertEquals(expectedFirstName1, actualRecords.get(1).getValueString("firstName")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryInput initQueryRequest() + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_PERSON); + return queryInput; + } + + + + /******************************************************************************* + ** This doesn't really test any RDBMS code, but is a checkpoint that the core + ** module is populating displayValues when it performs the system-level query action + ** (if so requested by input field). + *******************************************************************************/ + @Test + public void testThatDisplayValuesGetSetGoingThroughQueryAction() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setShouldGenerateDisplayValues(true); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + + for(QRecord record : queryOutput.getRecords()) + { + assertThat(record.getValues()).isNotEmpty(); + assertThat(record.getDisplayValues()).isNotEmpty(); + assertThat(record.getErrors()).isEmpty(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLookInsideTransaction() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + + InsertAction insertAction = new InsertAction(); + QBackendTransaction transaction = QBackendTransaction.openFor(insertInput); + + insertInput.setTransaction(transaction); + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "George").withValue("lastName", "Washington").withValue("email", "gw@kingsrook.com") + )); + + insertAction.execute(insertInput); + + QueryInput queryInput = initQueryRequest(); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Query without the transaction should not see the new row."); + + queryInput = initQueryRequest(); + queryInput.setTransaction(transaction); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(6, queryOutput.getRecords().size(), "Query with the transaction should see the new row."); + + transaction.rollback(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testEmptyInList() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.IN, List.of()))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "IN empty list should find nothing."); + + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.NOT_IN, List.of()))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "NOT_IN empty list should find everything."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOr() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "OR should find 2 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterAndOrOr() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Maes"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Complex query should find 2 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("lastName").equals("Maes")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("lastName").equals("Kelkhoff")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterOrAndAnd() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Chamberlain"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Complex query should find 1 row"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("lastName").equals("Chamberlain")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterAndTopLevelFilter() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 3)) + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Chamberlain"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Complex query should find 1 row"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueInteger("id").equals(3) && r.getValueString("firstName").equals("Tim") && r.getValueString("lastName").equals("Chamberlain")); + + queryInput.getFilter().setCriteria(List.of(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, 3))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "Next complex query should find 0 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithOrQueries() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.setFilter(new QQueryFilter( + new QFilterCriteria("billToPersonId", QCriteriaOperator.EQUALS, List.of(1)), + new QFilterCriteria("shipToPersonId", QCriteriaOperator.EQUALS, List.of(5)) + ).withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(5) + .allMatch(r -> Objects.equals(r.getValueInteger("billToPersonId"), 1) || Objects.equals(r.getValueInteger("shipToPersonId"), 5)); + + queryInput.setFilter(new QQueryFilter( + new QFilterCriteria("billToPersonId", QCriteriaOperator.EQUALS, List.of(1)), + new QFilterCriteria("shipToPersonId", QCriteriaOperator.EQUALS, List.of(5)) + ).withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .allMatch(r -> r.getValueInteger("storeId").equals(2)) + .allMatch(r -> Objects.equals(r.getValueInteger("billToPersonId"), 1) || Objects.equals(r.getValueInteger("shipToPersonId"), 5)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithSubFilters() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withSubFilters(List.of( + new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 2), new QFilterCriteria("billToPersonId", QCriteriaOperator.EQUALS, 1)), + new QQueryFilter(new QFilterCriteria("billToPersonId", QCriteriaOperator.IS_BLANK), new QFilterCriteria("shipToPersonId", QCriteriaOperator.IS_BLANK)).withBooleanOperator(QQueryFilter.BooleanOperator.OR) + ))); + Predicate p = r -> r.getValueInteger("billToPersonId") == null || r.getValueInteger("shipToPersonId") == null || (r.getValueInteger("id") >= 2 && r.getValueInteger("billToPersonId") == 1); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(4) + .allMatch(p); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .allMatch(r -> r.getValueInteger("storeId").equals(1)) + .allMatch(p); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(3) + .allMatch(r -> r.getValueInteger("storeId").equals(3)) + .allMatch(p); + + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityNullValues() throws Exception + { + runTestSql("INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (9, NULL, 1, 6)", null); + runTestSql("INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (10, NULL, 6, 5)", null); + + QInstance qInstance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + reInitInstanceInContext(qInstance); + + Predicate hasNullStoreId = r -> r.getValueInteger("storeId") == null; + + //////////////////////////////////////////// + // all-access user should get all 10 rows // + //////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(10) + .anyMatch(hasNullStoreId); + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // no-values user should get 0 rows (given that default null-behavior on this key type is DENY) // + ////////////////////////////////////////////////////////////////////////////////////////////////// + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // user with list of all ids shouldn't see the nulls (given that default null-behavior on this key type is DENY) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + { + QSession qSession = new QSession(); + for(Integer i : List.of(1, 2, 3, 4, 5)) + { + qSession.withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, i); + } + QContext.setQSession(qSession); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(8) + .noneMatch(hasNullStoreId); + } + + ////////////////////////////////////////////////////////////////////////// + // specifically set the null behavior to deny - repeat the last 2 tests // + ////////////////////////////////////////////////////////////////////////// + qInstance.getTable(TestUtils.TABLE_NAME_ORDER).getRecordSecurityLocks().get(0).setNullValueBehavior(RecordSecurityLock.NullValueBehavior.DENY); + + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + { + QSession qSession = new QSession(); + for(Integer i : List.of(1, 2, 3, 4, 5)) + { + qSession.withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, i); + } + QContext.setQSession(qSession); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(8) + .noneMatch(hasNullStoreId); + } + + /////////////////////////////////// + // change null behavior to ALLOW // + /////////////////////////////////// + qInstance.getTable(TestUtils.TABLE_NAME_ORDER).getRecordSecurityLocks().get(0).setNullValueBehavior(RecordSecurityLock.NullValueBehavior.ALLOW); + + ///////////////////////////////////////////// + // all-access user should still get all 10 // + ///////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(10) + .anyMatch(hasNullStoreId); + + ///////////////////////////////////////////////////// + // no-values user should only get the rows w/ null // + ///////////////////////////////////////////////////// + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .allMatch(hasNullStoreId); + + //////////////////////////////////////////////////// + // user with list of all ids should see the nulls // + //////////////////////////////////////////////////// + { + QSession qSession = new QSession(); + for(Integer i : List.of(1, 2, 3, 4, 5)) + { + qSession.withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, i); + } + QContext.setQSession(qSession); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(10) + .anyMatch(hasNullStoreId); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @SuppressWarnings("unchecked") + void testHeavyFields() throws QException + { + ////////////////////////////////////////////////////////// + // set homeTown field as heavy - so it won't be fetched // + ////////////////////////////////////////////////////////// + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON) + .getField("homeTown") + .withIsHeavy(true); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_PERSON); + List records = new QueryAction().execute(queryInput).getRecords(); + assertThat(records).describedAs("No records should have the heavy homeTown field set").noneMatch(r -> r.getValue("homeTown") != null); + assertThat(records).describedAs("Some records should have a homeTown length backend detail set").anyMatch(r -> ((Map) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") != null); + assertThat(records).describedAs("Some records should have a null homeTown length backend").anyMatch(r -> ((Map) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") == null); + + ////////////////////////////////////////////// + // re-do the query, requesting heavy fields // + ////////////////////////////////////////////// + queryInput.setShouldFetchHeavyFields(true); + records = new QueryAction().execute(queryInput).getRecords(); + assertThat(records).describedAs("Some records should have the heavy homeTown field set when heavies are requested").anyMatch(r -> r.getValue("homeTown") != null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNamesToInclude() throws QException + { + QQueryFilter filter = new QQueryFilter().withCriteria("id", QCriteriaOperator.EQUALS, 1); + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON).withFilter(filter); + + QRecord record = new QueryAction().execute(queryInput.withFieldNamesToInclude(null)).getRecords().get(0); + assertTrue(record.getValues().containsKey("id")); + assertTrue(record.getValues().containsKey("firstName")); + assertTrue(record.getValues().containsKey("createDate")); + assertEquals(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON).getFields().size(), record.getValues().size()); + + record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("id", "firstName"))).getRecords().get(0); + assertTrue(record.getValues().containsKey("id")); + assertTrue(record.getValues().containsKey("firstName")); + assertFalse(record.getValues().containsKey("createDate")); + assertEquals(2, record.getValues().size()); + + record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("homeTown"))).getRecords().get(0); + assertFalse(record.getValues().containsKey("id")); + assertFalse(record.getValues().containsKey("firstName")); + assertFalse(record.getValues().containsKey("createDate")); + assertEquals(1, record.getValues().size()); + } + +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteUpdateActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteUpdateActionTest.java new file mode 100644 index 00000000..bff3d231 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteUpdateActionTest.java @@ -0,0 +1,444 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteUpdateActionTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateNullList() throws QException + { + UpdateInput updateInput = initUpdateRequest(); + updateInput.setRecords(null); + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + assertEquals(0, updateResult.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateEmptyList() throws QException + { + UpdateInput updateInput = initUpdateRequest(); + updateInput.setRecords(Collections.emptyList()); + new UpdateAction().execute(updateInput); + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + assertEquals(0, updateResult.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateOne() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + QRecord record = new QRecord() + .withValue("id", 2) + .withValue("firstName", "James") + .withValue("lastName", "Kirk") + .withValue("email", "jamestk@starfleet.net") + .withValue("birthDate", "2210-05-20"); + updateInput.setRecords(List.of(record)); + + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + + assertEquals(1, updateResult.getRecords().size(), "Should return 1 row"); + assertEquals(2, updateResult.getRecords().get(0).getValue("id"), "Should have id=2 in the row"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE last_name = 'Kirk'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(2, rs.getInt("id")); + assertEquals("James", rs.getString("first_name")); + assertEquals("2210-05-20", rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Maes'", (rs -> + { + if(rs.next()) + { + fail("Should not have found Maes any more."); + } + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateManyWithDifferentColumnsAndValues() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + QRecord record1 = new QRecord() + .withValue("id", 1) + .withValue("firstName", "Darren") + .withValue("lastName", "From Bewitched") + .withValue("birthDate", "1900-01-01"); + + QRecord record2 = new QRecord() + .withValue("id", 3) + .withValue("firstName", "Wilt") + .withValue("birthDate", null); + + QRecord record3 = new QRecord() + .withValue("id", 5) + .withValue("firstName", "Richard") + .withValue("birthDate", null); + + updateInput.setRecords(List.of(record1, record2, record3)); + + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + + // this test runs one batch and one regular query + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(1, statistics.get(BaseRDBMSActionStrategy.STAT_BATCHES_RAN)); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + + assertEquals(3, updateResult.getRecords().size(), "Should return 3 rows"); + assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); + assertEquals(3, updateResult.getRecords().get(1).getValue("id"), "Should have expected ids in the row"); + assertEquals(5, updateResult.getRecords().get(2).getValue("id"), "Should have expected ids in the row"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE last_name = 'From Bewitched'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(1, rs.getInt("id")); + assertEquals("Darren", rs.getString("first_name")); + assertEquals("From Bewitched", rs.getString("last_name")); + assertEquals("1900-01-01", rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Chamberlain'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(3, rs.getInt("id")); + assertEquals("Wilt", rs.getString("first_name")); + assertNull(rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Richardson'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(5, rs.getInt("id")); + assertEquals("Richard", rs.getString("first_name")); + assertNull(rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateManyWithSameColumnsDifferentValues() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + QRecord record1 = new QRecord() + .withValue("id", 1) + .withValue("firstName", "Darren") + .withValue("lastName", "From Bewitched") + .withValue("birthDate", "1900-01-01"); + + QRecord record2 = new QRecord() + .withValue("id", 3) + .withValue("firstName", "Wilt") + .withValue("lastName", "Tim's Uncle") + .withValue("birthDate", null); + + updateInput.setRecords(List.of(record1, record2)); + + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(1, statistics.get(BaseRDBMSActionStrategy.STAT_BATCHES_RAN)); + + assertEquals(2, updateResult.getRecords().size(), "Should return 2 rows"); + assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); + assertEquals(3, updateResult.getRecords().get(1).getValue("id"), "Should have expected ids in the row"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE last_name = 'From Bewitched'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(1, rs.getInt("id")); + assertEquals("Darren", rs.getString("first_name")); + assertEquals("From Bewitched", rs.getString("last_name")); + assertEquals("1900-01-01", rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Tim''s Uncle'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(3, rs.getInt("id")); + assertEquals("Wilt", rs.getString("first_name")); + assertEquals("Tim's Uncle", rs.getString("last_name")); + assertNull(rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateManyWithSameColumnsSameValues() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + for(int i = 1; i <= 5; i++) + { + records.add(new QRecord() + .withValue("id", i) + .withValue("birthDate", "1999-09-09")); + } + + updateInput.setRecords(records); + + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + + assertEquals(5, updateResult.getRecords().size(), "Should return 5 rows"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE id <= 5", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals("1999-09-09", rs.getString("birth_date")); + } + assertEquals(5, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testModifyDateGetsUpdated() throws Exception + { + String originalModifyDate = selectModifyDate(1); + + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("id", 1) + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + new UpdateAction().execute(updateInput); + + String updatedModifyDate = selectModifyDate(1); + + assertTrue(StringUtils.hasContent(originalModifyDate)); + assertTrue(StringUtils.hasContent(updatedModifyDate)); + assertNotEquals(originalModifyDate, updatedModifyDate); + } + + + + /******************************************************************************* + ** This situation - fails in a real mysql, but not in h2... anyway, because mysql + ** didn't want to convert the date-time string format to a date-time. + *******************************************************************************/ + @Test + void testDateTimesCanBeModifiedFromIsoStrings() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("id", 1) + .withValue("createDate", "2022-10-03T10:29:35Z") + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + new UpdateAction().execute(updateInput); + } + + + + /******************************************************************************* + ** Make sure that records without a primary key come back with error. + *******************************************************************************/ + @Test + void testWithoutPrimaryKeyErrors() throws Exception + { + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + assertFalse(updateOutput.getRecords().get(0).getErrors().isEmpty()); + assertEquals("Missing value in primary key field", updateOutput.getRecords().get(0).getErrors().get(0).getMessage()); + } + + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("id", null) + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + assertFalse(updateOutput.getRecords().get(0).getErrors().isEmpty()); + assertEquals("Missing value in primary key field", updateOutput.getRecords().get(0).getErrors().get(0).getMessage()); + } + + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("id", null) + .withValue("firstName", "Johnny Not Updated")); + records.add(new QRecord() + .withValue("id", 2) + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + + assertFalse(updateOutput.getRecords().get(0).getErrors().isEmpty()); + assertEquals("Missing value in primary key field", updateOutput.getRecords().get(0).getErrors().get(0).getMessage()); + + assertTrue(updateOutput.getRecords().get(1).getErrors().isEmpty()); + + GetInput getInput = new GetInput(); + getInput.setTableName(TestUtils.TABLE_NAME_PERSON); + getInput.setPrimaryKey(2); + GetOutput getOutput = new GetAction().execute(getInput); + assertEquals("Johnny Updated", getOutput.getRecord().getValueString("firstName")); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String selectModifyDate(Integer id) throws Exception + { + StringBuilder modifyDate = new StringBuilder(); + runTestSql("SELECT modify_date FROM person WHERE id = " + id, (rs -> + { + if(rs.next()) + { + modifyDate.append(rs.getString("modify_date")); + } + })); + return (modifyDate.toString()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private UpdateInput initUpdateRequest() + { + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(TestUtils.TABLE_NAME_PERSON); + return updateInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-sqlite/src/test/resources/prime-test-database-parent-child-tables.sql b/qqq-backend-module-sqlite/src/test/resources/prime-test-database-parent-child-tables.sql new file mode 100644 index 00000000..e512a5d4 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/resources/prime-test-database-parent-child-tables.sql @@ -0,0 +1,49 @@ +-- +-- QQQ - Low-code Application Framework for Engineers. +-- Copyright (C) 2021-2025. Kingsrook, LLC +-- 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States +-- contact@kingsrook.com +-- https://github.com/Kingsrook/ +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . +-- + +DROP TABLE IF EXISTS parent_table; +DROP TABLE IF EXISTS child_table; + +CREATE TABLE child_table +( + id INT AUTO_INCREMENT primary key, + name VARCHAR(80) NOT NULL +); + +INSERT INTO child_table (id, name) VALUES (1, 'Timmy'); +INSERT INTO child_table (id, name) VALUES (2, 'Jimmy'); +INSERT INTO child_table (id, name) VALUES (3, 'Johnny'); +INSERT INTO child_table (id, name) VALUES (4, 'Gracie'); +INSERT INTO child_table (id, name) VALUES (5, 'Suzie'); + +CREATE TABLE parent_table +( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(80) NOT NULL, + child_id INT, + foreign key (child_id) references child_table(id) +); + +INSERT INTO parent_table (id, name, child_id) VALUES (1, 'Tim''s Dad', 1); +INSERT INTO parent_table (id, name, child_id) VALUES (2, 'Tim''s Mom', 1); +INSERT INTO parent_table (id, name, child_id) VALUES (3, 'Childless Man', null); +INSERT INTO parent_table (id, name, child_id) VALUES (4, 'Childless Woman', null); +INSERT INTO parent_table (id, name, child_id) VALUES (5, 'Johny''s Single Dad', 3); diff --git a/qqq-backend-module-sqlite/src/test/resources/prime-test-database.sql b/qqq-backend-module-sqlite/src/test/resources/prime-test-database.sql new file mode 100644 index 00000000..874f7583 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/resources/prime-test-database.sql @@ -0,0 +1,215 @@ +-- +-- QQQ - Low-code Application Framework for Engineers. +-- Copyright (C) 2021-2025. Kingsrook, LLC +-- 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States +-- contact@kingsrook.com +-- https://github.com/Kingsrook/ +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . +-- + +DROP TABLE IF EXISTS person; +CREATE TABLE person +( + id INTEGER PRIMARY KEY, + create_date TIMESTAMP, -- DEFAULT datetime('now'), -- can't get this to work! + modify_date TIMESTAMP, -- DEFAULT datetime('now'), + + first_name VARCHAR(80) NOT NULL, + last_name VARCHAR(80) NOT NULL, + birth_date DATE, + email VARCHAR(250) NOT NULL, + is_employed BOOLEAN, + annual_salary DECIMAL(12,2), + days_worked INTEGER, + home_town VARCHAR(80), + start_time TIME +); + +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 1, 25000, 27, 'Chester'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 1, 26000, 124, 'Chester'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 0, null, 0, 'Decatur'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 30000, 99, 'Texas'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1, 1000000, 232, null); + +DROP TABLE IF EXISTS personal_id_card; +CREATE TABLE personal_id_card +( + id INTEGER PRIMARY KEY, + create_date TIMESTAMP, -- DEFAULT date(), + modify_date TIMESTAMP, -- DEFAULT date(), + person_id INTEGER, + id_number VARCHAR(250) +); + +INSERT INTO personal_id_card (person_id, id_number) VALUES (1, '19800531'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (2, '19800515'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (3, '19760528'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (6, '123123123'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (null, '987987987'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (null, '456456456'); + +DROP TABLE IF EXISTS carrier; +CREATE TABLE carrier +( + id INTEGER PRIMARY KEY, + name VARCHAR(80) NOT NULL, + company_code VARCHAR(80) NOT NULL, + service_level VARCHAR(80) NOT NULL +); + +INSERT INTO carrier (id, name, company_code, service_level) VALUES (1, 'UPS Ground', 'UPS', 'G'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (2, 'UPS 2Day', 'UPS', '2'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (3, 'UPS International', 'UPS', 'I'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (4, 'Fedex Ground', 'FEDEX', 'G'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (5, 'Fedex Next Day', 'UPS', '1'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (6, 'Will Call', 'WILL_CALL', 'W'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (7, 'USPS Priority', 'USPS', '1'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (8, 'USPS Super Slow', 'USPS', '4'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (9, 'USPS Super Fast', 'USPS', '0'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (10, 'DHL International', 'DHL', 'I'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (11, 'GSO', 'GSO', 'G'); + +DROP TABLE IF EXISTS line_item_extrinsic; +DROP TABLE IF EXISTS order_line; +DROP TABLE IF EXISTS item; +DROP TABLE IF EXISTS `order`; +DROP TABLE IF EXISTS order_instructions; +DROP TABLE IF EXISTS warehouse_store_int; +DROP TABLE IF EXISTS store; +DROP TABLE IF EXISTS warehouse; + +CREATE TABLE store +( + id INTEGER PRIMARY KEY, + name VARCHAR(80) NOT NULL +); + +-- define 3 stores +INSERT INTO store (id, name) VALUES (1, 'Q-Mart'); +INSERT INTO store (id, name) VALUES (2, 'QQQ ''R'' Us'); +INSERT INTO store (id, name) VALUES (3, 'QDepot'); + +CREATE TABLE item +( + id INTEGER PRIMARY KEY, + sku VARCHAR(80) NOT NULL, + description VARCHAR(80), + store_id INT NOT NULL REFERENCES store +); + +-- three items for each store +INSERT INTO item (id, sku, description, store_id) VALUES (1, 'QM-1', 'Q-Mart Item 1', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (2, 'QM-2', 'Q-Mart Item 2', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (3, 'QM-3', 'Q-Mart Item 3', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (4, 'QRU-1', 'QQQ R Us Item 4', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (5, 'QRU-2', 'QQQ R Us Item 5', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (6, 'QRU-3', 'QQQ R Us Item 6', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (7, 'QD-1', 'QDepot Item 7', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (8, 'QD-2', 'QDepot Item 8', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (9, 'QD-3', 'QDepot Item 9', 3); + +CREATE TABLE `order` +( + id INTEGER PRIMARY KEY, + store_id INT REFERENCES store, + bill_to_person_id INT, + ship_to_person_id INT, + current_order_instructions_id INT -- f-key to order_instructions, which also has an f-key back here! +); + +-- variable orders +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (1, 1, 1, 1); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (2, 1, 1, 2); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (3, 1, 2, 3); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (4, 2, 4, 5); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (5, 2, 5, 4); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (6, 3, 5, null); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (7, 3, null, 5); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (8, 3, null, 5); + +CREATE TABLE order_instructions +( + id INTEGER PRIMARY KEY, + order_id INT, + instructions VARCHAR(250) +); + +-- give orders 1 & 2 multiple versions of the instruction record +INSERT INTO order_instructions (id, order_id, instructions) VALUES (1, 1, 'order 1 v1'); +INSERT INTO order_instructions (id, order_id, instructions) VALUES (2, 1, 'order 1 v2'); +UPDATE `order` SET current_order_instructions_id = 2 WHERE id=1; + +INSERT INTO order_instructions (id, order_id, instructions) VALUES (3, 2, 'order 2 v1'); +INSERT INTO order_instructions (id, order_id, instructions) VALUES (4, 2, 'order 2 v2'); +INSERT INTO order_instructions (id, order_id, instructions) VALUES (5, 2, 'order 2 v3'); +UPDATE `order` SET current_order_instructions_id = 5 WHERE id=2; + +-- give all other orders just 1 instruction +INSERT INTO order_instructions (order_id, instructions) SELECT id, concat('order ', id, ' v1') FROM `order` WHERE current_order_instructions_id IS NULL; +UPDATE `order` SET current_order_instructions_id = (SELECT MIN(id) FROM order_instructions WHERE order_id = `order`.id) WHERE current_order_instructions_id is null; + +CREATE TABLE order_line +( + id INTEGER PRIMARY KEY, + order_id INT REFERENCES `order`, + sku VARCHAR(80), + store_id INT REFERENCES store, + quantity INT +); + +-- various lines +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (1, 'QM-1', 1, 10); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (1, 'QM-2', 1, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (1, 'QM-3', 1, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (2, 'QRU-1', 2, 1); -- this line has an item from a different store than its order. +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (3, 'QM-1', 1, 20); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (4, 'QRU-1', 2, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (4, 'QRU-2', 2, 2); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (5, 'QRU-1', 2, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (6, 'QD-1', 3, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (7, 'QD-1', 3, 2); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (8, 'QD-1', 3, 3); + + +CREATE TABLE warehouse +( + id INTEGER PRIMARY KEY, + name VARCHAR(80) +); + +INSERT INTO warehouse (name) VALUES ('Patterson'); +INSERT INTO warehouse (name) VALUES ('Edison'); +INSERT INTO warehouse (name) VALUES ('Stockton'); +INSERT INTO warehouse (name) VALUES ('Somewhere in Texas'); + +CREATE TABLE warehouse_store_int +( + id INTEGER PRIMARY KEY, + warehouse_id INT REFERENCES `warehouse`, + store_id INT REFERENCES `store` +); + +INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 1); +INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 2); +INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 3); + +CREATE TABLE line_item_extrinsic +( + id INTEGER PRIMARY KEY, + order_line_id INT REFERENCES order_line, + `key` VARCHAR(80), + `value` VARCHAR(80) +); + diff --git a/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION b/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION index 2094a100..d21d277b 100644 --- a/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION +++ b/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION @@ -1 +1 @@ -0.24.0 +0.25.0 diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index c2c98d9b..4d24e89c 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -95,8 +95,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; @@ -1200,15 +1198,18 @@ public class QJavalinImplementation ///////////////////////////////////////////////////////////////////////////////////// if(backend != null && backend.getUsesVariants()) { - queryInput.setTableName(backend.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backend.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backend.getVariantOptionsTableTypeValue()))); + QTableMetaData variantsTable = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()); + + queryInput.setTableName(variantsTable.getName()); + queryInput.setFilter(backend.getBackendVariantsConfig().getOptionsFilter()); + queryInput.setShouldGenerateDisplayValues(true); QueryOutput output = new QueryAction().execute(queryInput); for(QRecord qRecord : output.getRecords()) { variants.add(new QFrontendVariant() - .withId(qRecord.getValue(backend.getVariantOptionsTableIdField())) - .withType(backend.getVariantOptionsTableTypeValue()) - .withName(qRecord.getValueString(backend.getVariantOptionsTableNameField()))); + .withId(qRecord.getValue(variantsTable.getPrimaryKeyField())) + .withType(backend.getBackendVariantsConfig().getVariantTypeKey()) + .withName(qRecord.getRecordLabel())); } QJavalinAccessLogger.logStartSilent("variants"); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 1ed7e826..c48582a1 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.PipedOutputStream; import java.io.Serializable; import java.time.LocalDate; @@ -50,24 +51,22 @@ import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessAction; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; -import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -79,9 +78,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMeta import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; @@ -93,6 +89,7 @@ import io.javalin.apibuilder.EndpointGroup; import io.javalin.http.Context; import io.javalin.http.UploadedFile; import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.BooleanUtils; import org.eclipse.jetty.http.HttpStatus; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; import static io.javalin.apibuilder.ApiBuilder.get; @@ -325,7 +322,7 @@ public class QJavalinProcessHandler *******************************************************************************/ public static void processInit(Context context) { - doProcessInitOrStep(context, null, null, RunProcessInput.FrontendStepBehavior.BREAK); + doProcessInitOrStep(context, null, null, null, RunProcessInput.FrontendStepBehavior.BREAK); } @@ -339,7 +336,7 @@ public class QJavalinProcessHandler *******************************************************************************/ public static void processRun(Context context) { - doProcessInitOrStep(context, null, null, RunProcessInput.FrontendStepBehavior.SKIP); + doProcessInitOrStep(context, null, null, null, RunProcessInput.FrontendStepBehavior.SKIP); } @@ -347,7 +344,7 @@ public class QJavalinProcessHandler /******************************************************************************* ** *******************************************************************************/ - private static void doProcessInitOrStep(Context context, String processUUID, String startAfterStep, RunProcessInput.FrontendStepBehavior frontendStepBehavior) + private static void doProcessInitOrStep(Context context, String processUUID, String startAfterStep, String startAtStep, RunProcessInput.FrontendStepBehavior frontendStepBehavior) { Map resultForCaller = new HashMap<>(); Exception returningException = null; @@ -362,8 +359,22 @@ public class QJavalinProcessHandler } resultForCaller.put("processUUID", processUUID); - LOG.info(startAfterStep == null ? "Initiating process [" + processName + "] [" + processUUID + "]" - : "Resuming process [" + processName + "] [" + processUUID + "] after step [" + startAfterStep + "]"); + if(startAfterStep == null && startAtStep == null) + { + LOG.info("Initiating process [" + processName + "] [" + processUUID + "]"); + } + else if(startAfterStep != null) + { + LOG.info("Resuming process [" + processName + "] [" + processUUID + "] after step [" + startAfterStep + "]"); + } + else if(startAtStep != null) + { + LOG.info("Resuming process [" + processName + "] [" + processUUID + "] at step [" + startAtStep + "]"); + } + else + { + LOG.warn("A logical impossibility was reached, regarding the nullity of startAfterStep and startAtStep, at least given how this code was originally written."); + } RunProcessInput runProcessInput = new RunProcessInput(); QJavalinImplementation.setupSession(context, runProcessInput); @@ -372,11 +383,13 @@ public class QJavalinProcessHandler runProcessInput.setFrontendStepBehavior(frontendStepBehavior); runProcessInput.setProcessUUID(processUUID); runProcessInput.setStartAfterStep(startAfterStep); + runProcessInput.setStartAtStep(startAtStep); populateRunProcessRequestWithValuesFromContext(context, runProcessInput); String reportName = ValueUtils.getValueAsString(runProcessInput.getValue("reportName")); QJavalinAccessLogger.logStart(startAfterStep == null ? "processInit" : "processStep", logPair("processName", processName), logPair("processUUID", processUUID), StringUtils.hasContent(startAfterStep) ? logPair("startAfterStep", startAfterStep) : null, + StringUtils.hasContent(startAtStep) ? logPair("startAtStep", startAfterStep) : null, StringUtils.hasContent(reportName) ? logPair("reportName", reportName) : null); ////////////////////////////////////////////////////////////////////////////////////////////////// @@ -485,6 +498,7 @@ public class QJavalinProcessHandler } resultForCaller.put("values", runProcessOutput.getValues()); runProcessOutput.getProcessState().getNextStepName().ifPresent(nextStep -> resultForCaller.put("nextStep", nextStep)); + runProcessOutput.getProcessState().getBackStepName().ifPresent(backStep -> resultForCaller.put("backStep", backStep)); ///////////////////////////////////////////////////////////////////////////////////////////////////////////// // todo - delete after all frontends look for processMetaDataAdjustment instead of updatedFrontendStepList // @@ -531,7 +545,7 @@ public class QJavalinProcessHandler ** todo - make query params have a "field-" type of prefix?? ** *******************************************************************************/ - private static void populateRunProcessRequestWithValuesFromContext(Context context, RunProcessInput runProcessInput) throws IOException + private static void populateRunProcessRequestWithValuesFromContext(Context context, RunProcessInput runProcessInput) throws IOException, QException { ////////////////////////// // process query string // @@ -562,20 +576,42 @@ public class QJavalinProcessHandler //////////////////////////// // process uploaded files // //////////////////////////// - for(UploadedFile uploadedFile : context.uploadedFiles()) + for(Map.Entry> entry : context.uploadedFileMap().entrySet()) { - try(InputStream content = uploadedFile.content()) + String name = entry.getKey(); + List uploadedFiles = entry.getValue(); + ArrayList storageInputs = new ArrayList<>(); + runProcessInput.addValue(name, storageInputs); + + String storageTableName = QJavalinImplementation.javalinMetaData.getUploadedFileArchiveTableName(); + if(!StringUtils.hasContent(storageTableName)) { - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes(content.readAllBytes()); - qUploadedFile.setFilename(uploadedFile.filename()); + throw (new QException("UploadFileArchiveTableName was not specified in javalinMetaData. Cannot accept file uploads.")); + } - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - LOG.info("Stored uploaded file in TempFileStateProvider under key: " + key); - runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); + for(UploadedFile uploadedFile : uploadedFiles) + { + String reference = QValueFormatter.formatDate(LocalDate.now()) + + File.separator + runProcessInput.getProcessName() + + File.separator + UUID.randomUUID() + + File.separator + uploadedFile.filename(); - archiveUploadedFile(runProcessInput, qUploadedFile); + StorageInput storageInput = new StorageInput(storageTableName).withReference(reference); + storageInputs.add(storageInput); + + try + ( + InputStream content = uploadedFile.content(); + OutputStream outputStream = new StorageAction().createOutputStream(storageInput); + ) + { + content.transferTo(outputStream); + LOG.info("Streamed uploaded file", logPair("storageTable", storageTableName), logPair("reference", reference), logPair("processName", runProcessInput.getProcessName()), logPair("uploadFileName", uploadedFile.filename())); + } + catch(QException e) + { + throw (new QException("Error creating output stream in table [" + storageTableName + "] for storage action", e)); + } } } @@ -606,27 +642,6 @@ public class QJavalinProcessHandler - /******************************************************************************* - ** - *******************************************************************************/ - private static void archiveUploadedFile(RunProcessInput runProcessInput, QUploadedFile qUploadedFile) - { - String fileName = QValueFormatter.formatDate(LocalDate.now()) - + File.separator + runProcessInput.getProcessName() - + File.separator + qUploadedFile.getFilename(); - - InsertInput insertInput = new InsertInput(); - insertInput.setTableName(QJavalinImplementation.javalinMetaData.getUploadedFileArchiveTableName()); - insertInput.setRecords(List.of(new QRecord() - .withValue("fileName", fileName) - .withValue("contents", qUploadedFile.getBytes()) - )); - - new InsertAction().executeAsync(insertInput); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -684,8 +699,19 @@ public class QJavalinProcessHandler public static void processStep(Context context) { String processUUID = context.pathParam("processUUID"); - String lastStep = context.pathParam("step"); - doProcessInitOrStep(context, processUUID, lastStep, RunProcessInput.FrontendStepBehavior.BREAK); + + String startAfterStep = null; + String startAtStep = null; + if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(context.queryParam("isStepBack")))) + { + startAtStep = context.pathParam("step"); + } + else + { + startAfterStep = context.pathParam("step"); + } + + doProcessInitOrStep(context, processUUID, startAfterStep, startAtStep, RunProcessInput.FrontendStepBehavior.BREAK); } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java index f0f2c413..92971c63 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java @@ -77,7 +77,8 @@ public class QApplicationJavalinServer private boolean serveLegacyUnversionedMiddlewareAPI = true; private List middlewareVersionList = List.of(new MiddlewareVersionV1()); private List additionalRouteProviders = null; - private Consumer javalinConfigurationCustomizer = null; + private Consumer javalinConfigurationCustomizer = null; + private QJavalinMetaData javalinMetaData = null; private long lastQInstanceHotSwapMillis; private long millisBetweenHotSwaps = 2500; @@ -115,6 +116,11 @@ public class QApplicationJavalinServer { if(serveFrontendMaterialDashboard) { + if(getClass().getResource("/material-dashboard/index.html") == null) + { + LOG.warn("/material-dashboard/index.html resource was not found. This might happen if you're using a local (e.g., within-IDE) snapshot version... Try updating pom.xml to reference a released version of qfmd?"); + } + //////////////////////////////////////////////////////////////////////////////////////// // If you have any assets to add to the web server (e.g., logos, icons) place them at // // src/main/resources/material-dashboard-overlay // @@ -149,7 +155,7 @@ public class QApplicationJavalinServer { try { - QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); + QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance, javalinMetaData); config.router.apiBuilder(qJavalinImplementation.getRoutes()); } catch(QInstanceValidationException e) @@ -623,4 +629,35 @@ public class QApplicationJavalinServer return (this); } + + /******************************************************************************* + ** Getter for javalinMetaData + *******************************************************************************/ + public QJavalinMetaData getJavalinMetaData() + { + return (this.javalinMetaData); + } + + + + /******************************************************************************* + ** Setter for javalinMetaData + *******************************************************************************/ + public void setJavalinMetaData(QJavalinMetaData javalinMetaData) + { + this.javalinMetaData = javalinMetaData; + } + + + + /******************************************************************************* + ** Fluent setter for javalinMetaData + *******************************************************************************/ + public QApplicationJavalinServer withJavalinMetaData(QJavalinMetaData javalinMetaData) + { + this.javalinMetaData = javalinMetaData; + return (this); + } + + } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java index c138b0bc..372b9aac 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java @@ -45,6 +45,7 @@ public class ProcessInitOrStepInput extends AbstractMiddlewareInput ///////////////////////////////////// private String processUUID; private String startAfterStep; + // todo - add (in next version?) startAtStep (for back) private RunProcessInput.FrontendStepBehavior frontendStepBehavior = RunProcessInput.FrontendStepBehavior.BREAK; diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java index f7d0d4a5..aef6ab44 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java @@ -58,6 +58,8 @@ public interface ProcessInitOrStepOrStatusOutputInterface extends AbstractMiddle *******************************************************************************/ void setNextStep(String nextStep); + // todo - add (in next version?) backStep + /******************************************************************************* ** Setter for values *******************************************************************************/ diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java index 9b0de98c..a97a7019 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java @@ -38,6 +38,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIHasAdditionalProperties; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIIncludeProperties; @@ -123,7 +124,25 @@ public class SchemaBuilder if(c.isEnum()) { schema.withType(Type.STRING); - schema.withEnumValues(Arrays.stream(c.getEnumConstants()).map(e -> String.valueOf(e)).collect(Collectors.toList())); + + if(element.isAnnotationPresent(OpenAPIEnumSubSet.class)) + { + try + { + OpenAPIEnumSubSet enumSubSetAnnotation = element.getAnnotation(OpenAPIEnumSubSet.class); + Class> enumSubSetClass = enumSubSetAnnotation.value(); + OpenAPIEnumSubSet.EnumSubSet enumSubSetContainer = enumSubSetClass.getConstructor().newInstance(); + schema.withEnumValues(enumSubSetContainer.getSubSet().stream().map(e -> String.valueOf(e)).collect(Collectors.toList())); + } + catch(Exception e) + { + throw new QRuntimeException("Error processing OpenAPIEnumSubSet on element: " + element, e); + } + } + else + { + schema.withEnumValues(Arrays.stream(c.getEnumConstants()).map(e -> String.valueOf(e)).collect(Collectors.toList())); + } } else if(c.equals(String.class)) { diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java new file mode 100644 index 00000000..aeebca15 --- /dev/null +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java @@ -0,0 +1,116 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; + + +import java.io.Serializable; +import java.util.EnumSet; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FieldAdornment implements ToSchema +{ + @OpenAPIExclude() + private com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment wrapped; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public FieldAdornment(com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment wrapped) + { + this.wrapped = wrapped; + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public FieldAdornment() + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class FieldAdornmentSubSet implements OpenAPIEnumSubSet.EnumSubSet + { + private static EnumSet subSet = null; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EnumSet getSubSet() + { + if(subSet == null) + { + EnumSet subSet = EnumSet.allOf(AdornmentType.class); + subSet.remove(AdornmentType.FILE_UPLOAD); // todo - remove for next version! + subSet.remove(AdornmentType.TOOLTIP); // todo - remove for next version! + FieldAdornmentSubSet.subSet = subSet; + } + + return (subSet); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @OpenAPIDescription("Type of this adornment") + @OpenAPIEnumSubSet(FieldAdornmentSubSet.class) + public AdornmentType getType() + { + return (this.wrapped == null || this.wrapped.getType() == null ? null : this.wrapped.getType()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @OpenAPIDescription("Values associated with this adornment. Keys and the meanings of their values will differ by type.") + public Map getValues() + { + return (this.wrapped.getValues()); + } + +} diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java index 5c4505ac..b8a75d5f 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java @@ -23,7 +23,6 @@ package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; import java.util.List; -import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; @@ -61,6 +60,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -82,6 +82,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -92,6 +93,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -102,6 +104,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -112,6 +115,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -122,6 +126,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -143,6 +148,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -153,6 +159,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -166,6 +173,8 @@ public class FieldMetaData implements ToSchema // todo - inline PVS + + /*************************************************************************** ** ***************************************************************************/ @@ -177,14 +186,17 @@ public class FieldMetaData implements ToSchema // todo behaviors? + + + /*************************************************************************** ** ***************************************************************************/ @OpenAPIDescription("Special UI dressings to add to the field.") - @OpenAPIListItems(value = FieldAdornment.class) // todo! + @OpenAPIListItems(value = FieldAdornment.class, useRef = true) public List getAdornments() { - return (this.wrapped.getAdornments()); + return (this.wrapped.getAdornments() == null ? null : this.wrapped.getAdornments().stream().map(a -> new FieldAdornment(a)).toList()); } // todo help content diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java index e28af1ce..f6c15758 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java @@ -23,11 +23,13 @@ package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; import java.io.Serializable; +import java.util.EnumSet; import java.util.Map; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIMapKnownEntries; @@ -63,10 +65,39 @@ public class FrontendComponent implements ToSchema + /*************************************************************************** + ** + ***************************************************************************/ + public static class QComponentTypeSubSet implements OpenAPIEnumSubSet.EnumSubSet + { + private static EnumSet subSet = null; + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EnumSet getSubSet() + { + if(subSet == null) + { + EnumSet subSet = EnumSet.allOf(QComponentType.class); + subSet.remove(QComponentType.BULK_LOAD_FILE_MAPPING_FORM); + subSet.remove(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM); + subSet.remove(QComponentType.BULK_LOAD_PROFILE_FORM); + QComponentTypeSubSet.subSet = subSet; + } + + return(subSet); + } + } + + + /*************************************************************************** ** ***************************************************************************/ @OpenAPIDescription("The type of this component. e.g., what kind of UI element(s) should be presented to the user.") + @OpenAPIEnumSubSet(QComponentTypeSubSet.class) public QComponentType getType() { return (this.wrapped.getType()); diff --git a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml index e6772436..b0aebaf4 100644 --- a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml +++ b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml @@ -130,26 +130,31 @@ components: description: "Description of the error" type: "string" type: "object" + FieldAdornment: + properties: + type: + description: "Type of this adornment" + enum: + - "LINK" + - "CHIP" + - "SIZE" + - "CODE_EDITOR" + - "RENDER_HTML" + - "REVEAL" + - "FILE_DOWNLOAD" + - "ERROR" + type: "string" + values: + description: "Values associated with this adornment. Keys and the meanings\ + \ of their values will differ by type." + type: "object" + type: "object" FieldMetaData: properties: adornments: description: "Special UI dressings to add to the field." items: - properties: - type: - enum: - - "LINK" - - "CHIP" - - "SIZE" - - "CODE_EDITOR" - - "RENDER_HTML" - - "REVEAL" - - "FILE_DOWNLOAD" - - "ERROR" - type: "string" - values: - type: "object" - type: "object" + $ref: "#/components/schemas/FieldAdornment" type: "array" defaultValue: description: "Default value to use in this field." @@ -973,21 +978,7 @@ components: adornments: description: "Special UI dressings to add to the field." items: - properties: - type: - enum: - - "LINK" - - "CHIP" - - "SIZE" - - "CODE_EDITOR" - - "RENDER_HTML" - - "REVEAL" - - "FILE_DOWNLOAD" - - "ERROR" - type: "string" - values: - type: "object" - type: "object" + $ref: "#/components/schemas/FieldAdornment" type: "array" defaultValue: description: "Default value to use in this field." @@ -1483,6 +1474,8 @@ paths: processes: person.bulkInsert: hasPermission: true + icon: + name: "library_add" isHidden: true label: "Person Bulk Insert" name: "person.bulkInsert" diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java index 560ddb0b..588fa284 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java @@ -40,8 +40,10 @@ import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceLambda; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; @@ -1382,6 +1384,7 @@ class QJavalinImplementationTest extends QJavalinTestBase Function makeNewInstanceWithBackendName = (backendName) -> { QInstance newInstance = new QInstance(); + newInstance.setAuthentication(new QAuthenticationMetaData().withType(QAuthenticationType.FULLY_ANONYMOUS).withName("anonymous")); newInstance.addBackend(new QBackendMetaData().withName(backendName).withBackendType(MockBackendModule.class)); if(!"invalid".equals(backendName)) diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index 3d23493d..42224882 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -68,17 +68,36 @@ com.kingsrook.qqq qqq-frontend-material-dashboard - 0.24.0-SNAPSHOT + 0.24.0 com.h2database h2 2.2.220 - test - + + + + org.slf4j + slf4j-simple + 2.0.6 + + + + + org.seleniumhq.selenium + selenium-java + 4.19.1 + test + + + io.github.bonigarcia + webdrivermanager + 5.6.2 + test + diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index 03a3c934..bc40d1a1 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -22,7 +22,10 @@ package com.kingsrook.sampleapp.metadata; +import java.io.InputStream; import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -49,8 +52,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMe import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -59,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -71,9 +80,12 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinali import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.sampleapp.dashboard.widgets.PersonsByCreateDateBarChart; import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; +import org.apache.commons.io.IOUtils; /******************************************************************************* @@ -81,7 +93,7 @@ import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; *******************************************************************************/ public class SampleMetaDataProvider extends AbstractQQQApplication { - public static boolean USE_MYSQL = true; + public static boolean USE_MYSQL = false; public static final String RDBMS_BACKEND_NAME = "rdbms"; public static final String FILESYSTEM_BACKEND_NAME = "filesystem"; @@ -98,6 +110,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive"; public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_PET = "pet"; public static final String TABLE_NAME_CARRIER = "carrier"; public static final String TABLE_NAME_CITY = "city"; @@ -109,7 +122,6 @@ public class SampleMetaDataProvider extends AbstractQQQApplication - /*************************************************************************** ** ***************************************************************************/ @@ -120,6 +132,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication } + /******************************************************************************* ** *******************************************************************************/ @@ -132,6 +145,10 @@ public class SampleMetaDataProvider extends AbstractQQQApplication qInstance.addBackend(defineFilesystemBackend()); qInstance.addTable(defineTableCarrier()); qInstance.addTable(defineTablePerson()); + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(TABLE_NAME_PERSON)); + qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum(PetSpecies.NAME, PetSpecies.values())); + qInstance.addTable(defineTablePet()); + qInstance.addJoin(defineTablePersonJoinPet()); qInstance.addTable(defineTableCityFile()); qInstance.addProcess(defineProcessGreetPeople()); qInstance.addProcess(defineProcessGreetPeopleInteractive()); @@ -151,6 +168,26 @@ public class SampleMetaDataProvider extends AbstractQQQApplication + /******************************************************************************* + ** + *******************************************************************************/ + public static void primeTestDatabase(String sqlFileName) throws Exception + { + try(Connection connection = ConnectionManager.getConnection(SampleMetaDataProvider.defineRdbmsBackend())) + { + InputStream primeTestDatabaseSqlStream = SampleMetaDataProvider.class.getResourceAsStream("/" + sqlFileName); + List lines = IOUtils.readLines(primeTestDatabaseSqlStream, StandardCharsets.UTF_8); + lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); + String joinedSQL = String.join("\n", lines); + for(String sql : joinedSQL.split(";")) + { + QueryManager.executeUpdate(connection, sql); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -204,6 +241,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication .withIcon(new QIcon().withName("emoji_people")) .withChild(qInstance.getProcess(PROCESS_NAME_GREET).withIcon(new QIcon().withName("emoji_people"))) .withChild(qInstance.getTable(TABLE_NAME_PERSON).withIcon(new QIcon().withName("person"))) + .withChild(qInstance.getTable(TABLE_NAME_PET).withIcon(new QIcon().withName("pets"))) .withChild(qInstance.getTable(TABLE_NAME_CITY).withIcon(new QIcon().withName("location_city"))) .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE).withIcon(new QIcon().withName("waving_hand"))) .withWidgets(List.of(PersonsByCreateDateBarChart.class.getSimpleName(), QuickSightChartRenderer.class.getSimpleName())) @@ -340,7 +378,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name").withIsRequired(true)) .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name").withIsRequired(true)) .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withField(new QFieldMetaData("email", QFieldType.STRING).withIsRequired(true)) .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN).withBackendName("is_employed")) .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary").withDisplayFormat(DisplayFormat.CURRENCY)) .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked").withDisplayFormat(DisplayFormat.COMMAS)) @@ -352,11 +390,62 @@ public class SampleMetaDataProvider extends AbstractQQQApplication QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData); + qTableMetaData.withAssociation(new Association() + .withAssociatedTableName(TABLE_NAME_PET) + .withName("pets") + .withJoinName(QJoinMetaData.makeInferredJoinName(TABLE_NAME_PERSON, TABLE_NAME_PET))); + return (qTableMetaData); } + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineTablePet() + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(TABLE_NAME_PET) + .withLabel("Pet") + .withBackendName(RDBMS_BACKEND_NAME) + .withPrimaryKeyField("id") + .withRecordLabelFormat("%s %s") + .withRecordLabelFields("name") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false)) + .withField(new QFieldMetaData("name", QFieldType.STRING).withBackendName("name").withIsRequired(true)) + .withField(new QFieldMetaData("personId", QFieldType.INTEGER).withBackendName("person_id").withIsRequired(true).withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("speciesId", QFieldType.INTEGER).withBackendName("species_id").withIsRequired(true).withPossibleValueSourceName(PetSpecies.NAME)) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + + .withSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name"))) + .withSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("personId", "speciesId", "birthDate"))) + .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData); + + return (qTableMetaData); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QJoinMetaData defineTablePersonJoinPet() + { + return new QJoinMetaData() + .withLeftTable(TABLE_NAME_PERSON) + .withRightTable(TABLE_NAME_PET) + .withInferredName() + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "personId")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -390,7 +479,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication .withLabel("Greet People") .withTableName(TABLE_NAME_PERSON) .withIsHidden(true) - .addStep(new QBackendStepMetaData() + .withStep(new QBackendStepMetaData() .withName("prepare") .withCode(new QCodeReference(MockBackendStep.class)) .withInputData(new QFunctionInputMetaData() @@ -419,16 +508,16 @@ public class SampleMetaDataProvider extends AbstractQQQApplication .withName(PROCESS_NAME_GREET_INTERACTIVE) .withTableName(TABLE_NAME_PERSON) - .addStep(LoadInitialRecordsStep.defineMetaData(TABLE_NAME_PERSON)) + .withStep(LoadInitialRecordsStep.defineMetaData(TABLE_NAME_PERSON)) - .addStep(new QFrontendStepMetaData() + .withStep(new QFrontendStepMetaData() .withName("setup") .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) .withFormField(new QFieldMetaData("greetingPrefix", QFieldType.STRING)) .withFormField(new QFieldMetaData("greetingSuffix", QFieldType.STRING)) ) - .addStep(new QBackendStepMetaData() + .withStep(new QBackendStepMetaData() .withName("doWork") .withCode(new QCodeReference() .withName(MockBackendStep.class.getName()) @@ -447,7 +536,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ) - .addStep(new QFrontendStepMetaData() + .withStep(new QFrontendStepMetaData() .withName("results") .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)) @@ -499,7 +588,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication return new QProcessMetaData() .withName(PROCESS_NAME_SIMPLE_SLEEP) .withIsHidden(true) - .addStep(SleeperStep.getMetaData()); + .withStep(SleeperStep.getMetaData()); } @@ -511,12 +600,12 @@ public class SampleMetaDataProvider extends AbstractQQQApplication { return new QProcessMetaData() .withName(PROCESS_NAME_SLEEP_INTERACTIVE) - .addStep(new QFrontendStepMetaData() + .withStep(new QFrontendStepMetaData() .withName(SCREEN_0) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) .withFormField(new QFieldMetaData("outputMessage", QFieldType.STRING))) - .addStep(SleeperStep.getMetaData()) - .addStep(new QFrontendStepMetaData() + .withStep(SleeperStep.getMetaData()) + .withStep(new QFrontendStepMetaData() .withName(SCREEN_1) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) .withFormField(new QFieldMetaData("outputMessage", QFieldType.STRING))); @@ -531,7 +620,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication { return new QProcessMetaData() .withName(PROCESS_NAME_SIMPLE_THROW) - .addStep(ThrowerStep.getMetaData()); + .withStep(ThrowerStep.getMetaData()); } @@ -637,4 +726,53 @@ public class SampleMetaDataProvider extends AbstractQQQApplication } } + + + /*************************************************************************** + ** + ***************************************************************************/ + public enum PetSpecies implements PossibleValueEnum + { + DOG(1, "Dog"), + CAT(2, "Cat"); + + private final Integer id; + private final String label; + + public static final String NAME = "petSpecies"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + PetSpecies(int id, String label) + { + this.id = id; + this.label = label; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Integer getPossibleValueId() + { + return (id); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return (label); + } + } + } diff --git a/qqq-sample-project/src/test/resources/prime-test-database.sql b/qqq-sample-project/src/main/resources/prime-test-database.sql similarity index 82% rename from qqq-sample-project/src/test/resources/prime-test-database.sql rename to qqq-sample-project/src/main/resources/prime-test-database.sql index 10185156..1f5e6bc9 100644 --- a/qqq-sample-project/src/test/resources/prime-test-database.sql +++ b/qqq-sample-project/src/main/resources/prime-test-database.sql @@ -42,6 +42,27 @@ INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, a INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 950000, 75); INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 0, 1500000, 1); +DROP TABLE IF EXISTS pet; +CREATE TABLE pet +( + id INT AUTO_INCREMENT primary key , + create_date TIMESTAMP DEFAULT now(), + modify_date TIMESTAMP DEFAULT now(), + + name VARCHAR(80) NOT NULL, + species_id INTEGER NOT NULL, + person_id INTEGER NOT NULL, + birth_date DATE +); + +INSERT INTO pet (id, name, species_id, person_id) VALUES (1, 'Charlie', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (2, 'Coco', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (3, 'Louie', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (4, 'Barkley', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (5, 'Toby', 1, 2); +INSERT INTO pet (id, name, species_id, person_id) VALUES (6, 'Mae', 2, 3); + + DROP TABLE IF EXISTS carrier; CREATE TABLE carrier ( diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java new file mode 100644 index 00000000..c0e1c1b7 --- /dev/null +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java @@ -0,0 +1,101 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.sampleapp.selenium; + + +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.sampleapp.SampleJavalinServer; +import org.junit.jupiter.api.BeforeEach; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BaseSampleSeleniumTest // extends QBaseSeleniumTest +{ + private static final QLogger LOG = QLogger.getLogger(BaseSampleSeleniumTest.class); + + public static final Integer DEFAULT_WAIT_SECONDS = 10; + + private int port = 8011; + + + ///******************************************************************************* + // ** + // *******************************************************************************/ + //@Override + //@BeforeEach + //public void beforeEach() + //{ + // super.beforeEach(); + // qSeleniumLib.withBaseUrl("http://localhost:" + port); + // qSeleniumLib.withWaitSeconds(DEFAULT_WAIT_SECONDS); + // + // new SampleJavalinServer().startJavalinServer(port); + //} + // + // + // + ///******************************************************************************* + // ** + // *******************************************************************************/ + //@Override + //protected boolean useInternalJavalin() + //{ + // return (false); + //} + // + // + // + // + ///******************************************************************************* + // ** + // *******************************************************************************/ + //public void clickLeftNavMenuItem(String text) + //{ + // qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiListItem-root", text).click(); + //} + // + // + // + ///******************************************************************************* + // ** + // *******************************************************************************/ + //public void clickLeftNavMenuItemThenSubItem(String text, String subItemText) + //{ + // qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiListItem-root", text).click(); + // qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiCollapse-vertical.MuiCollapse-entered .MuiListItem-root", subItemText).click(); + //} + // + // + // + ///******************************************************************************* + // ** + // *******************************************************************************/ + //public void goToPathAndWaitForSelectorContaining(String path, String selector, String text) + //{ + // driver.get(qSeleniumLib.getBaseUrl() + path); + // qSeleniumLib.waitForSelectorContaining(selector, text); + //} + +} + diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java new file mode 100644 index 00000000..90fc67b8 --- /dev/null +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.sampleapp.selenium; + + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadSeleniumTest extends BaseSampleSeleniumTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @Disabled("selenium not working in circleci at this time...") + void testSimple() throws IOException + { + String email = "jtkirk@starfleet.com"; + String tablePath = "/peopleApp/greetingsApp/person"; + + //////////////////////////////////// + // write a file to be bulk-loaded // + //////////////////////////////////// + String path = "/tmp/" + UUID.randomUUID() + ".csv"; + String csv = String.format(""" + email,firstName,lastName + %s,James T.,Kirk + """, email); + FileUtils.writeStringToFile(new File(path), csv, StandardCharsets.UTF_8); + + //goToPathAndWaitForSelectorContaining(tablePath + "/person.bulkInsert", ".MuiTypography-h5", "Person Bulk Insert: Upload File"); + // + //////////////////////////////// + //// complete the upload form // + //////////////////////////////// + //qSeleniumLib.waitForSelector("input[type=file]").sendKeys(path); + //qSeleniumLib.waitForSelectorContaining("button", "next").click(); + // + /////////////////////////////////////////// + //// proceed through file-mapping screen // + /////////////////////////////////////////// + //qSeleniumLib.waitForSelectorContaining("button", "next").click(); + // + //////////////////////////////////////////////////// + //// confirm data on preview screen, then proceed // + //////////////////////////////////////////////////// + //qSeleniumLib.waitForSelectorContaining("form#review .MuiTypography-body2 div", email); + //qSeleniumLib.waitForSelectorContaining("form#review .MuiTypography-body2 div", "Preview 1 of 1"); + //qSeleniumLib.waitForSelectorContaining("button", "arrow_forward").click(); // to avoid the record-preview 'next' button + // + ///////////////////////////////////////// + //// proceed through validation screen // + ///////////////////////////////////////// + //qSeleniumLib.waitForSelectorContaining("button", "submit").click(); + // + ////////////////////////////////////////// + //// confirm result screen and close it // + ////////////////////////////////////////// + //qSeleniumLib.waitForSelectorContaining(".MuiListItemText-root", "1 Person record was inserted"); + //qSeleniumLib.waitForSelectorContaining("button", "close").click(); + // + ////////////////////////////////////////////// + //// go to the order that was just inserted // + //// bonus - also test record-view-by-key // + ////////////////////////////////////////////// + //goToPathAndWaitForSelectorContaining(tablePath + "/key?email=" + email, "h5", "Viewing Person"); + } + +}