QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updates for more heavy-field handling

Browse Source
This commit is contained in:
darin.kelkhoff
2023-05-30 10:13:04 -05:00
parent 8235bb2bb7
commit a75530b466
13 changed files with 362 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
View File

@ -29,8 +29,10 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -43,8 +45,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
@ -112,9 +116,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
////////////////////////////////////////////////////////////////////////////
// build the list of fields that will be processed in the result-set loop //
////////////////////////////////////////////////////////////////////////////
List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values().stream()
.filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields()))
.toList());
List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values().stream().toList());
for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
{
if(queryJoin.getSelect())
@ -123,10 +125,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
for(QFieldMetaData joinField : joinTable.getFields().values())
{
if(filterOutHeavyFieldsIfNeeded(joinField, queryInput.getShouldFetchHeavyFields()))
{
fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName()));
}
fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName()));
}
}
}
@ -153,9 +152,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
for(int i = 1; i <= metaData.getColumnCount(); i++)
{
QFieldMetaData qFieldMetaData = fieldList.get(i - 1);
Serializable value = getFieldValueFromResultSet(qFieldMetaData, resultSet, i);
values.put(qFieldMetaData.getName(), value);
QFieldMetaData field = fieldList.get(i - 1);
if(!queryInput.getShouldFetchHeavyFields() && field.getIsHeavy())
{
///////////////////////////////////////////////////////////////////////////////////
// if this is a non-fetched heavy field (e.g., we just fetched its length), then //
// get the value here as an INTEGER, not a BLOB or whatever the field would be //
///////////////////////////////////////////////////////////////////////////////////
Serializable fieldLength = getFieldValueFromResultSet(QFieldType.INTEGER, resultSet, i);
setHeavyFieldLengthInRecordBackendDetails(record, field, fieldLength);
}
else
{
Serializable value = getFieldValueFromResultSet(field, resultSet, i);
values.put(field.getName(), value);
}
}
queryOutput.addRecord(record);
@ -195,6 +207,27 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
private static void setHeavyFieldLengthInRecordBackendDetails(QRecord record, QFieldMetaData field, Serializable fieldLength)
{
if(record.getBackendDetails() == null)
{
record.setBackendDetails(new HashMap<>());
}
if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null)
{
record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>());
}
((Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(field.getName(), fieldLength);
}
/*******************************************************************************
**
*******************************************************************************/
@ -210,8 +243,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values());
String columns = fieldList.stream()
.filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields()))
.map(field -> escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field)))
.map(field -> Pair.of(field, escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field))))
.map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
.collect(Collectors.joining(", "));
StringBuilder rs = new StringBuilder(clausePrefix).append(columns);
@ -229,7 +262,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
List<QFieldMetaData> joinFieldList = new ArrayList<>(joinTable.getFields().values());
String joinColumns = joinFieldList.stream()
.filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields()))
.map(field -> escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field)))
.map(field -> Pair.of(field, escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field))))
.map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
.collect(Collectors.joining(", "));
rs.append(", ").append(joinColumns);
}
@ -254,6 +288,24 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
/*******************************************************************************
** if we're not fetching heavy fields, instead just get their length. this
** method wraps the field 'sql name' (e.g., column_name or table_name.column_name)
** with the LENGTH() function, if needed.
*******************************************************************************/
private String wrapHeavyFieldsWithLengthFunctionIfNeeded(Pair<QFieldMetaData, String> fieldAndSqlName, boolean shouldFetchHeavyFields)
{
QFieldMetaData field = fieldAndSqlName.getA();
String sqlName = fieldAndSqlName.getB();
if(!shouldFetchHeavyFields && field.getIsHeavy())
{
return ("LENGTH(" + sqlName + ")");
}
return (sqlName);
}
/*******************************************************************************
**
*******************************************************************************/

1
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java
View File

@ -167,6 +167,7 @@ public class TestUtils
.withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN).withBackendName("is_employed"))
.withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary"))
.withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked"))
.withField(new QFieldMetaData("homeTown", QFieldType.STRING).withBackendName("home_town"))
.withBackendDetails(new RDBMSTableBackendDetails()
.withTableName("person"));
}

33
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java
View File

@ -22,10 +22,12 @@
package com.kingsrook.qqq.backend.module.rdbms.actions;
import java.io.Serializable;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@ -1663,4 +1665,35 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
.hasSize(1);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testHeavyFields() throws QException
{
//////////////////////////////////////////////////////////
// set homeTown field as heavy - so it won't be fetched //
//////////////////////////////////////////////////////////
QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON)
.getField("homeTown")
.withIsHeavy(true);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_PERSON);
List<QRecord> records = new QueryAction().execute(queryInput).getRecords();
assertThat(records).describedAs("No records should have the heavy homeTown field set").noneMatch(r -> r.getValue("homeTown") != null);
assertThat(records).describedAs("Some records should have a homeTown length backend detail set").anyMatch(r -> ((Map<String, Serializable>) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") != null);
assertThat(records).describedAs("Some records should have a null homeTown length backend").anyMatch(r -> ((Map<String, Serializable>) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") == null);
//////////////////////////////////////////////
// re-do the query, requesting heavy fields //
//////////////////////////////////////////////
queryInput.setShouldFetchHeavyFields(true);
records = new QueryAction().execute(queryInput).getRecords();
assertThat(records).describedAs("Some records should have the heavy homeTown field set when heavies are requested").anyMatch(r -> r.getValue("homeTown") != null);
}
}

13
qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql
View File

@ -32,14 +32,15 @@ CREATE TABLE person
email VARCHAR(250) NOT NULL,
is_employed BOOLEAN,
annual_salary DECIMAL(12,2),
days_worked INTEGER
days_worked INTEGER,
home_town VARCHAR(80)
);
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 1, 25000, 27);
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 1, 26000, 124);
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 0, null, 0);
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 30000, 99);
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1, 1000000, 232);
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 1, 25000, 27, 'Chester');
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 1, 26000, 124, 'Chester');
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 0, null, 0, 'Decatur');
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 30000, 99, 'Texas');
INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1, 1000000, 232, null);
DROP TABLE IF EXISTS personal_id_card;
CREATE TABLE personal_id_card