Search for join meta data through exposed joins.

This commit is contained in:
2023-05-01 14:17:08 -05:00
parent e0e4519708
commit 5070502bde
5 changed files with 413 additions and 26 deletions

View File

@ -32,12 +32,16 @@ import java.util.Objects;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MutableList;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -46,6 +50,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
*******************************************************************************/
public class JoinsContext
{
private static final QLogger LOG = QLogger.getLogger(JoinsContext.class);
private final QInstance instance;
private final String mainTableName;
private final List<QueryJoin> queryJoins;
@ -65,7 +71,7 @@ public class JoinsContext
{
this.instance = instance;
this.mainTableName = tableName;
this.queryJoins = CollectionUtils.nonNullList(queryJoins);
this.queryJoins = new MutableList<>(queryJoins);
for(QueryJoin queryJoin : this.queryJoins)
{
@ -123,6 +129,8 @@ public class JoinsContext
ensureFilterIsRepresented(filter);
addJoinsFromExposedJoinPaths();
/* todo!!
for(QueryJoin queryJoin : queryJoins)
{
@ -132,7 +140,147 @@ public class JoinsContext
// addCriteriaForRecordSecurityLock(instance, session, joinTable, securityCriteria, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias());
}
}
*/
*/
}
/*******************************************************************************
** If there are any joins in the context that don't have a join meta data, see
** if we can find the JoinMetaData to use for them by looking at the main table's
** exposed joins, and using their join paths.
*******************************************************************************/
private void addJoinsFromExposedJoinPaths() throws QException
{
////////////////////////////////////////////////////////////////////////////////
// do a double-loop, to avoid concurrent modification on the queryJoins list. //
// that is to say, we'll loop over that list, but possibly add things to it, //
// in which case we'll set this flag, and break the inner loop, to go again. //
////////////////////////////////////////////////////////////////////////////////
boolean addedJoin;
do
{
addedJoin = false;
for(QueryJoin queryJoin : queryJoins)
{
/////////////////////////////////////////////////////////////////////
// if the join has joinMetaData, then we don't need to process it. //
/////////////////////////////////////////////////////////////////////
if(queryJoin.getJoinMetaData() == null)
{
//////////////////////////////////////////////////////////////////////
// try to find a direct join between the main table and this table. //
// if one is found, then put it (the meta data) on the query join. //
//////////////////////////////////////////////////////////////////////
String baseTableName = Objects.requireNonNullElse(resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), mainTableName);
QJoinMetaData found = findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable());
if(found != null)
{
queryJoin.setJoinMetaData(found);
}
else
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// else, the join must be indirect - so look for an exposedJoin that will have a joinPath that will connect us //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
LOG.debug("Looking for an exposed join...", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()));
QTableMetaData mainTable = instance.getTable(mainTableName);
boolean addedAnyQueryJoins = false;
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins()))
{
if(queryJoin.getJoinTable().equals(exposedJoin.getJoinTable()))
{
LOG.debug("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath()));
/////////////////////////////////////////////////////////////////////////////////////
// loop backward through the join path (from the joinTable back to the main table) //
// adding joins to the table (if they aren't already in the query) //
/////////////////////////////////////////////////////////////////////////////////////
String tmpTable = queryJoin.getJoinTable();
for(int i = exposedJoin.getJoinPath().size() - 1; i >= 0; i--)
{
String joinName = exposedJoin.getJoinPath().get(i);
QJoinMetaData joinToAdd = instance.getJoin(joinName);
/////////////////////////////////////////////////////////////////////////////
// get the name from the opposite side of the join (flipping it if needed) //
/////////////////////////////////////////////////////////////////////////////
String nextTable;
if(joinToAdd.getRightTable().equals(tmpTable))
{
nextTable = joinToAdd.getLeftTable();
}
else
{
nextTable = joinToAdd.getRightTable();
joinToAdd = joinToAdd.flip();
}
if(doesJoinNeedAddedToQuery(joinName))
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if this is the last element in the joinPath, then we want to set this joinMetaData on the outer queryJoin //
// - else, we need to add a new queryJoin to this context //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(i == exposedJoin.getJoinPath().size() - 1)
{
if(queryJoin.getBaseTableOrAlias() == null)
{
queryJoin.setBaseTableOrAlias(nextTable);
}
queryJoin.setJoinMetaData(joinToAdd);
}
else
{
QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd);
queryJoinToAdd.setType(queryJoin.getType());
addedAnyQueryJoins = true;
this.queryJoins.add(queryJoinToAdd); // todo something else with aliases? probably.
processQueryJoin(queryJoinToAdd);
}
}
tmpTable = nextTable;
}
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// break the inner loop (it would fail due to a concurrent modification), but continue the outer //
///////////////////////////////////////////////////////////////////////////////////////////////////
if(addedAnyQueryJoins)
{
addedJoin = true;
break;
}
}
}
}
}
while(addedJoin);
}
/*******************************************************************************
**
*******************************************************************************/
private boolean doesJoinNeedAddedToQuery(String joinName)
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
// look at all queryJoins already in context - if any have this join's name, then we don't need this join... //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(QueryJoin queryJoin : queryJoins)
{
if(queryJoin.getJoinMetaData() != null && queryJoin.getJoinMetaData().getName().equals(joinName))
{
return (false);
}
}
return (true);
}
@ -256,34 +404,54 @@ public class JoinsContext
{
if(!aliasToTableNameMap.containsKey(filterTable) && !Objects.equals(mainTableName, filterTable))
{
boolean found = false;
for(QJoinMetaData join : CollectionUtils.nonNullMap(QContext.getQInstance().getJoins()).values())
{
QueryJoin queryJoin = null;
if(join.getLeftTable().equals(mainTableName) && join.getRightTable().equals(filterTable))
{
queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
}
else
{
join = join.flip();
if(join.getLeftTable().equals(mainTableName) && join.getRightTable().equals(filterTable))
{
queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
}
}
QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join);
if(queryJoin != null)
{
this.queryJoins.add(queryJoin); // todo something else with aliases? probably.
processQueryJoin(queryJoin);
found = true;
break;
}
}
if(!found)
{
QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER);
this.queryJoins.add(queryJoin); // todo something else with aliases? probably.
processQueryJoin(queryJoin);
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private QueryJoin makeQueryJoinFromJoinAndTableNames(String tableA, String tableB, QJoinMetaData join)
{
QueryJoin queryJoin = null;
if(join.getLeftTable().equals(tableA) && join.getRightTable().equals(tableB))
{
queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
}
else
{
join = join.flip();
if(join.getLeftTable().equals(tableA) && join.getRightTable().equals(tableB))
{
queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
}
}
return queryJoin;
}
/*******************************************************************************
**
*******************************************************************************/

View File

@ -30,9 +30,12 @@ import java.time.Instant;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
@ -203,7 +206,8 @@ public abstract class AbstractRDBMSAction implements QActionInterface
{
StringBuilder rs = new StringBuilder(escapeIdentifier(getTableName(instance.getTable(tableName))) + " AS " + escapeIdentifier(tableName));
for(QueryJoin queryJoin : joinsContext.getQueryJoins())
List<QueryJoin> queryJoins = sortQueryJoinsForFromClause(tableName, joinsContext.getQueryJoins());
for(QueryJoin queryJoin : queryJoins)
{
QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable());
String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
@ -264,6 +268,48 @@ public abstract class AbstractRDBMSAction implements QActionInterface
/*******************************************************************************
** We've seen some SQL dialects (mysql, but not h2...) be unhappy if we have
** the from/join-ons out of order. This method resorts the joins, to start with
** main table, then any tables attached to it, then fanning out from there.
*******************************************************************************/
private List<QueryJoin> sortQueryJoinsForFromClause(String mainTableName, List<QueryJoin> queryJoins)
{
List<QueryJoin> inputListCopy = new ArrayList<>(queryJoins);
List<QueryJoin> rs = new ArrayList<>();
Set<String> seenTables = new HashSet<>();
seenTables.add(mainTableName);
boolean keepGoing = true;
while(!inputListCopy.isEmpty() && keepGoing)
{
keepGoing = false;
Iterator<QueryJoin> iterator = inputListCopy.iterator();
while(iterator.hasNext())
{
QueryJoin next = iterator.next();
if((StringUtils.hasContent(next.getBaseTableOrAlias()) && seenTables.contains(next.getBaseTableOrAlias())) || seenTables.contains(next.getJoinTable()))
{
rs.add(next);
if(StringUtils.hasContent(next.getBaseTableOrAlias()))
{
seenTables.add(next.getBaseTableOrAlias());
}
seenTables.add(next.getJoinTable());
iterator.remove();
keepGoing = true;
}
}
}
rs.addAll(inputListCopy);
return (rs);
}
/*******************************************************************************
** method that sub-classes should call to make a full WHERE clause, including
** security clauses.
@ -902,6 +948,16 @@ public abstract class AbstractRDBMSAction implements QActionInterface
/*******************************************************************************
** Make it easy (e.g., for tests) to turn on logging of SQL
*******************************************************************************/
public static void setLogSQLOutput(String loggerOrSystemOut)
{
System.setProperty("qqq.rdbms.logSQL.output", loggerOrSystemOut);
}
/*******************************************************************************
**
*******************************************************************************/

View File

@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
@ -239,6 +240,7 @@ public class TestUtils
qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER, "order")
.withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId"))
.withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine"))
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem")))
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE))
.withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
.withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
@ -246,7 +248,9 @@ public class TestUtils
qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item")
.withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId"))
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderLineJoinItem", "orderJoinOrderLine")))
.withField(new QFieldMetaData("sku", QFieldType.STRING))
.withField(new QFieldMetaData("description", QFieldType.STRING))
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE))
);

View File

@ -51,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -72,6 +73,20 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
public void beforeEach() throws Exception
{
super.primeTestDatabase();
// AbstractRDBMSAction.setLogSQL(true);
// AbstractRDBMSAction.setLogSQLOutput("system.out");
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
void afterEach()
{
AbstractRDBMSAction.setLogSQL(false);
}
@ -1103,6 +1118,149 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
/*******************************************************************************
** Given tables:
** order - orderLine - item
** with exposedJoin on order to item
** do a query on order, also selecting item.
*******************************************************************************/
@Test
void testTwoTableAwayExposedJoin() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
QInstance instance = TestUtils.defineInstance();
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
queryInput.withQueryJoins(List.of(
new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true)
));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
List<QRecord> records = queryOutput.getRecords();
assertThat(records).hasSize(11); // one per line item
assertThat(records).allMatch(r -> r.getValue("id") != null);
assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null);
}
/*******************************************************************************
** Given tables:
** order - orderLine - item
** with exposedJoin on item to order
** do a query on item, also selecting order.
** This is a reverse of the above, to make sure join flipping, etc, is good.
*******************************************************************************/
@Test
void testTwoTableAwayExposedJoinReversed() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
QInstance instance = TestUtils.defineInstance();
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_ITEM);
queryInput.withQueryJoins(List.of(
new QueryJoin(TestUtils.TABLE_NAME_ORDER).withType(QueryJoin.Type.INNER).withSelect(true)
));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
List<QRecord> records = queryOutput.getRecords();
assertThat(records).hasSize(11); // one per line item
assertThat(records).allMatch(r -> r.getValue("description") != null);
assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER + ".id") != null);
}
/*******************************************************************************
** Given tables:
** order - orderLine - item
** with exposedJoin on order to item
** do a query on order, also selecting item, and also selecting orderLine...
*******************************************************************************/
@Test
void testTwoTableAwayExposedJoinAlsoSelectingInBetweenTable() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
QInstance instance = TestUtils.defineInstance();
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
queryInput.withQueryJoins(List.of(
new QueryJoin(TestUtils.TABLE_NAME_ORDER_LINE).withType(QueryJoin.Type.INNER).withSelect(true),
new QueryJoin(TestUtils.TABLE_NAME_ITEM).withType(QueryJoin.Type.INNER).withSelect(true)
));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
List<QRecord> records = queryOutput.getRecords();
assertThat(records).hasSize(11); // one per line item
assertThat(records).allMatch(r -> r.getValue("id") != null);
assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity") != null);
assertThat(records).allMatch(r -> r.getValue(TestUtils.TABLE_NAME_ITEM + ".description") != null);
}
/*******************************************************************************
** Given tables:
** order - orderLine - item
** with exposedJoin on order to item
** do a query on order, filtered by item
*******************************************************************************/
@Test
void testTwoTableAwayExposedJoinWhereClauseOnly() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
QInstance instance = TestUtils.defineInstance();
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
List<QRecord> records = queryOutput.getRecords();
assertThat(records).hasSize(4);
assertThat(records).allMatch(r -> r.getValue("id") != null);
}
/*******************************************************************************
** Given tables:
** order - orderLine - item
** with exposedJoin on order to item
** do a query on order, filtered by item
*******************************************************************************/
@Test
void testTwoTableAwayExposedJoinWhereClauseBothJoinTables() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
QInstance instance = TestUtils.defineInstance();
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
queryInput.setFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ITEM + ".description", QCriteriaOperator.STARTS_WITH, "Q-Mart"))
.withCriteria(new QFilterCriteria(TestUtils.TABLE_NAME_ORDER_LINE + ".quantity", QCriteriaOperator.IS_NOT_BLANK))
);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
List<QRecord> records = queryOutput.getRecords();
assertThat(records).hasSize(4);
assertThat(records).allMatch(r -> r.getValue("id") != null);
}
/*******************************************************************************
** queries on the store table, where the primary key (id) is the security field
*******************************************************************************/

View File

@ -102,19 +102,20 @@ CREATE TABLE item
(
id INT AUTO_INCREMENT PRIMARY KEY,
sku VARCHAR(80) NOT NULL,
description VARCHAR(80),
store_id INT NOT NULL REFERENCES store
);
-- three items for each store
INSERT INTO item (id, sku, store_id) VALUES (1, 'QM-1', 1);
INSERT INTO item (id, sku, store_id) VALUES (2, 'QM-2', 1);
INSERT INTO item (id, sku, store_id) VALUES (3, 'QM-3', 1);
INSERT INTO item (id, sku, store_id) VALUES (4, 'QRU-1', 2);
INSERT INTO item (id, sku, store_id) VALUES (5, 'QRU-2', 2);
INSERT INTO item (id, sku, store_id) VALUES (6, 'QRU-3', 2);
INSERT INTO item (id, sku, store_id) VALUES (7, 'QD-1', 3);
INSERT INTO item (id, sku, store_id) VALUES (8, 'QD-2', 3);
INSERT INTO item (id, sku, store_id) VALUES (9, 'QD-3', 3);
INSERT INTO item (id, sku, description, store_id) VALUES (1, 'QM-1', 'Q-Mart Item 1', 1);
INSERT INTO item (id, sku, description, store_id) VALUES (2, 'QM-2', 'Q-Mart Item 2', 1);
INSERT INTO item (id, sku, description, store_id) VALUES (3, 'QM-3', 'Q-Mart Item 3', 1);
INSERT INTO item (id, sku, description, store_id) VALUES (4, 'QRU-1', 'QQQ R Us Item 4', 2);
INSERT INTO item (id, sku, description, store_id) VALUES (5, 'QRU-2', 'QQQ R Us Item 5', 2);
INSERT INTO item (id, sku, description, store_id) VALUES (6, 'QRU-3', 'QQQ R Us Item 6', 2);
INSERT INTO item (id, sku, description, store_id) VALUES (7, 'QD-1', 'QDepot Item 7', 3);
INSERT INTO item (id, sku, description, store_id) VALUES (8, 'QD-2', 'QDepot Item 8', 3);
INSERT INTO item (id, sku, description, store_id) VALUES (9, 'QD-3', 'QDepot Item 9', 3);
CREATE TABLE `order`
(