diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/assessment/Assessable.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/assessment/Assessable.java new file mode 100644 index 00000000..48ba3802 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/assessment/Assessable.java @@ -0,0 +1,37 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.assessment; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; + + +/******************************************************************************* + ** marker for an object which can be processed by the QInstanceAssessor. + *******************************************************************************/ +public interface Assessable +{ + /*************************************************************************** + ** + ***************************************************************************/ + void assess(QInstanceAssessor qInstanceAssessor, QInstance qInstance); +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/assessment/QInstanceAssessor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/assessment/QInstanceAssessor.java new file mode 100644 index 00000000..08b25dfa --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/assessment/QInstanceAssessor.java @@ -0,0 +1,215 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.assessment; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** POC of a class that is meant to review meta-data for accuracy vs. real backends. + *******************************************************************************/ +public class QInstanceAssessor +{ + private static final QLogger LOG = QLogger.getLogger(QInstanceAssessor.class); + + private final QInstance qInstance; + + private List errors = new ArrayList<>(); + private List warnings = new ArrayList<>(); + private List suggestions = new ArrayList<>(); + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QInstanceAssessor(QInstance qInstance) + { + this.qInstance = qInstance; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void assess() + { + for(QBackendMetaData backend : qInstance.getBackends().values()) + { + if(backend instanceof Assessable assessable) + { + assessable.assess(this, qInstance); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("checkstyle:AvoidEscapedUnicodeCharacters") + public void printSummary() + { + /////////////////////////// + // print header & errors // + /////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(errors)) + { + System.out.println("Assessment passed with no errors! \uD83D\uDE0E"); + } + else + { + System.out.println("Assessment found the following " + StringUtils.plural(errors, "error", "errors") + ": \uD83D\uDE32"); + + for(String error : errors) + { + System.out.println(" - " + error); + } + } + + ///////////////////////////////////// + // print warnings if there are any // + ///////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(warnings)) + { + System.out.println("\nAssessment found the following " + StringUtils.plural(warnings, "warning", "warnings") + ": \uD83E\uDD28"); + + for(String warning : warnings) + { + System.out.println(" - " + warning); + } + } + + ////////////////////////////////////////// + // print suggestions, if there were any // + ////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(suggestions)) + { + System.out.println("\nThe following " + StringUtils.plural(suggestions, "fix is", "fixes are") + " suggested: \uD83E\uDD13"); + + for(String suggestion : suggestions) + { + System.out.println("\n" + suggestion + "\n"); + } + } + } + + + + /******************************************************************************* + ** Getter for qInstance + ** + *******************************************************************************/ + public QInstance getInstance() + { + return qInstance; + } + + + + /******************************************************************************* + ** Getter for errors + ** + *******************************************************************************/ + public List getErrors() + { + return errors; + } + + + + /******************************************************************************* + ** Getter for warnings + ** + *******************************************************************************/ + public List getWarnings() + { + return warnings; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addError(String errorMessage) + { + errors.add(errorMessage); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addWarning(String warningMessage) + { + warnings.add(warningMessage); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addError(String errorMessage, Exception e) + { + addError(errorMessage + " : " + e.getMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addSuggestion(String message) + { + suggestions.add(message); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public int getExitCode() + { + if(CollectionUtils.nullSafeHasContents(errors)) + { + return (1); + } + else + { + return (0); + } + } +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index e5511ed8..2bd74f71 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -111,7 +111,7 @@ public abstract class AbstractRDBMSAction ** ** That is, table.backendDetails.tableName if set -- else, table.name *******************************************************************************/ - protected String getTableName(QTableMetaData table) + public static String getTableName(QTableMetaData table) { if(table.getBackendDetails() instanceof RDBMSTableBackendDetails details) { @@ -130,7 +130,7 @@ public abstract class AbstractRDBMSAction ** ** That is, field.backendName if set -- else, field.name *******************************************************************************/ - protected String getColumnName(QFieldMetaData field) + public static String getColumnName(QFieldMetaData field) { if(field.getBackendName() != null) { diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendAssessor.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendAssessor.java new file mode 100644 index 00000000..720fb8cf --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendAssessor.java @@ -0,0 +1,331 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.instances.assessment.QInstanceAssessor; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.rdbms.actions.AbstractRDBMSAction; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RDBMSBackendAssessor +{ + private QInstanceAssessor assessor; + private RDBMSBackendMetaData backendMetaData; + private List tables; + + private Map typeMap = new HashMap<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public RDBMSBackendAssessor(QInstanceAssessor assessor, RDBMSBackendMetaData backendMetaData, List tables) + { + this.assessor = assessor; + this.backendMetaData = backendMetaData; + this.tables = tables; + + //////////////////////////////////////////////// + // these are types as returned by mysql // + // let null in here mean unsupported QQQ type // + //////////////////////////////////////////////// + typeMap.put("TEXT", QFieldType.TEXT); + typeMap.put("BINARY", QFieldType.BLOB); + typeMap.put("SET", null); + typeMap.put("VARBINARY", QFieldType.BLOB); + typeMap.put("MEDIUMBLOB", QFieldType.BLOB); + typeMap.put("NUMERIC", QFieldType.INTEGER); + typeMap.put("BIGINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("MEDIUMINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("SMALLINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("TINYINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("BIT", null); + typeMap.put("FLOAT", null); + typeMap.put("REAL", null); + typeMap.put("VARCHAR", QFieldType.STRING); + typeMap.put("BOOL", QFieldType.BOOLEAN); + typeMap.put("YEAR", null); + typeMap.put("TIME", QFieldType.TIME); + typeMap.put("TIMESTAMP", QFieldType.DATE_TIME); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void assess() + { + try(Connection connection = new ConnectionManager().getConnection(backendMetaData)) + { + //////////////////////////////////////////////////////////////////// + // read data type ids (integers) to names, for field-type mapping // + //////////////////////////////////////////////////////////////////// + DatabaseMetaData databaseMetaData; + Map dataTypeMap = new HashMap<>(); + try + { + databaseMetaData = connection.getMetaData(); + ResultSet typeInfoResultSet = databaseMetaData.getTypeInfo(); + while(typeInfoResultSet.next()) + { + String name = typeInfoResultSet.getString("TYPE_NAME"); + Integer id = typeInfoResultSet.getInt("DATA_TYPE"); + dataTypeMap.put(id, name); + } + } + catch(Exception e) + { + assessor.addError("Error loading metaData from RDBMS for backendName: " + backendMetaData.getName() + " - assessment cannot be completed.", e); + return; + } + + /////////////////////////////////////// + // process each table in the backend // + /////////////////////////////////////// + for(QTableMetaData table : tables) + { + String tableName = AbstractRDBMSAction.getTableName(table); + + try + { + /////////////////////////////// + // check if the table exists // + /////////////////////////////// + String databaseName = backendMetaData.getDatabaseName(); // these work for mysql - unclear about other vendors. + String schemaName = null; + try(ResultSet tableResultSet = databaseMetaData.getTables(databaseName, schemaName, tableName, null)) + { + if(!tableResultSet.next()) + { + assessor.addError("Table: " + table.getName() + " was not found in backend: " + backendMetaData.getName()); + assessor.addSuggestion(suggestCreateTable(table)); + continue; + } + + ////////////////////////////// + // read the table's columns // + ////////////////////////////// + Map columnMap = new HashMap<>(); + String primaryKeyColumnName = null; + try(ResultSet columnsResultSet = databaseMetaData.getColumns(databaseName, schemaName, tableName, null)) + { + while(columnsResultSet.next()) + { + String columnName = columnsResultSet.getString("COLUMN_NAME"); + String columnSize = columnsResultSet.getString("COLUMN_SIZE"); + Integer dataTypeId = columnsResultSet.getInt("DATA_TYPE"); + String isNullable = columnsResultSet.getString("IS_NULLABLE"); + String isAutoIncrement = columnsResultSet.getString("IS_AUTOINCREMENT"); + + String dataTypeName = dataTypeMap.get(dataTypeId); + QFieldMetaData columnMetaData = new QFieldMetaData(columnName, typeMap.get(dataTypeName)); + columnMap.put(columnName, columnMetaData); + + if("YES" .equals(isAutoIncrement)) + { + primaryKeyColumnName = columnName; + } + } + } + + ///////////////////////////////// + // diff the columns and fields // + ///////////////////////////////// + for(QFieldMetaData column : columnMap.values()) + { + boolean fieldExists = table.getFields().values().stream().anyMatch(f -> column.getName().equals(AbstractRDBMSAction.getColumnName(f))); + if(!fieldExists) + { + assessor.addWarning("Table: " + table.getName() + " has a column which was not found in the metaData: " + column.getName()); + assessor.addSuggestion("// in QTableMetaData.withName(\"" + table.getName() + "\")\n" + + ".withField(new QFieldMetaData(\"" + column.getName() + "\", QFieldType." + column.getType() + ").withBackendName(\"" + column.getName() + "\")"); // todo - column_name to fieldName + } + } + + for(QFieldMetaData field : table.getFields().values()) + { + String columnName = AbstractRDBMSAction.getColumnName(field); + boolean columnExists = columnMap.values().stream().anyMatch(c -> c.getName().equals(columnName)); + if(!columnExists) + { + assessor.addError("Table: " + table.getName() + " has a field which was not found in the database: " + field.getName()); + assessor.addSuggestion("/* For table [" + tableName + "] in backend [" + table.getBackendName() + " (database " + databaseName + ")]: */\n" + + "ALTER TABLE " + tableName + " ADD " + QInstanceEnricher.inferBackendName(columnName) + " " + getDatabaseTypeForField(table, field) + ";"); + } + } + + /////////////////////////////////////////////// + // read unique constraints from the database // + /////////////////////////////////////////////// + Map> uniqueIndexMap = new HashMap<>(); + try(ResultSet indexInfoResultSet = databaseMetaData.getIndexInfo(databaseName, schemaName, tableName, true, true)) + { + while(indexInfoResultSet.next()) + { + String indexName = indexInfoResultSet.getString("INDEX_NAME"); + String columnName = indexInfoResultSet.getString("COLUMN_NAME"); + uniqueIndexMap.computeIfAbsent(indexName, k -> new HashSet<>()); + uniqueIndexMap.get(indexName).add(columnName); + } + } + + ////////////////////////// + // diff the unique keys // + ////////////////////////// + for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys())) + { + Set fieldNames = uniqueKey.getFieldNames().stream().map(fieldName -> AbstractRDBMSAction.getColumnName(table.getField(fieldName))).collect(Collectors.toSet()); + if(!uniqueIndexMap.containsValue(fieldNames)) + { + assessor.addWarning("Table: " + table.getName() + " specifies a uniqueKey which was not found in the database: " + uniqueKey.getFieldNames()); + assessor.addSuggestion("/* For table [" + tableName + "] in backend [" + table.getBackendName() + " (database " + databaseName + ")]: */\n" + + "ALTER TABLE " + tableName + " ADD UNIQUE (" + StringUtils.join(", ", fieldNames) + ");"); + } + } + + for(Set uniqueIndex : uniqueIndexMap.values()) + { + ////////////////////////// + // skip the primary key // + ////////////////////////// + if(uniqueIndex.size() == 1 && uniqueIndex.contains(primaryKeyColumnName)) + { + continue; + } + + boolean foundInTableMetaData = false; + for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys())) + { + Set fieldNames = uniqueKey.getFieldNames().stream().map(fieldName -> AbstractRDBMSAction.getColumnName(table.getField(fieldName))).collect(Collectors.toSet()); + if(uniqueIndex.equals(fieldNames)) + { + foundInTableMetaData = true; + break; + } + } + + if(!foundInTableMetaData) + { + assessor.addWarning("Table: " + table.getName() + " has a unique index which was not found in the metaData: " + uniqueIndex); + assessor.addSuggestion("// in QTableMetaData.withName(\"" + table.getName() + "\")\n" + + ".withUniqueKey(new UniqueKey(\"" + StringUtils.join("\", \"", uniqueIndex) + "\"))"); + } + } + + } + } + catch(Exception e) + { + assessor.addError("Error assessing table: " + table.getName() + " in backend: " + backendMetaData.getName(), e); + } + } + } + catch(Exception e) + { + assessor.addError("Error connecting to RDBMS for backendName: " + backendMetaData.getName(), e); + return; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String suggestCreateTable(QTableMetaData table) + { + StringBuilder rs = new StringBuilder("/* For table [" + table.getName() + "] in backend [" + table.getBackendName() + " (database " + (backendMetaData.getDatabaseName()) + ")]: */\n"); + rs.append("CREATE TABLE ").append(AbstractRDBMSAction.getTableName(table)).append("\n"); + rs.append("(\n"); + + List fields = new ArrayList<>(); + for(QFieldMetaData field : table.getFields().values()) + { + fields.add(" " + AbstractRDBMSAction.getColumnName(field) + " " + getDatabaseTypeForField(table, field)); + } + + rs.append(StringUtils.join(",\n", fields)); + + rs.append("\n);"); + return (rs.toString()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getDatabaseTypeForField(QTableMetaData table, QFieldMetaData field) + { + return switch(field.getType()) + { + case STRING -> + { + int n = Objects.requireNonNullElse(field.getMaxLength(), 250); + yield ("VARCHAR(" + n + ")"); + } + case INTEGER -> + { + String suffix = table.getPrimaryKeyField().equals(field.getName()) ? " AUTO_INCREMENT PRIMARY KEY" : ""; + yield ("INTEGER" + suffix); + } + case LONG -> + { + String suffix = table.getPrimaryKeyField().equals(field.getName()) ? " AUTO_INCREMENT PRIMARY KEY" : ""; + yield ("BIGINT" + suffix); + } + case DECIMAL -> "DECIMAL(10,2)"; + case BOOLEAN -> "BOOLEAN"; + case DATE -> "DATE"; + case TIME -> "TIME"; + case DATE_TIME -> "TIMESTAMP"; + case TEXT -> "TEXT"; + case HTML -> "TEXT"; + case PASSWORD -> "VARCHAR(40)"; + case BLOB -> "BLOB"; + }; + } +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java index 277502ed..20796d2f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java @@ -22,12 +22,18 @@ package com.kingsrook.qqq.backend.module.rdbms.model.metadata; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.instances.assessment.Assessable; +import com.kingsrook.qqq.backend.core.instances.assessment.QInstanceAssessor; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule; import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; @@ -36,7 +42,7 @@ import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterf /******************************************************************************* ** Meta-data to provide details of an RDBMS backend (e.g., connection params) *******************************************************************************/ -public class RDBMSBackendMetaData extends QBackendMetaData +public class RDBMSBackendMetaData extends QBackendMetaData implements Assessable { private String vendor; private String hostName; @@ -580,4 +586,25 @@ public class RDBMSBackendMetaData extends QBackendMetaData } + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void assess(QInstanceAssessor qInstanceAssessor, QInstance qInstance) + { + List tables = new ArrayList<>(); + for(QTableMetaData table : qInstance.getTables().values()) + { + if(Objects.equals(getName(), table.getBackendName())) + { + tables.add(table); + } + } + + if(!tables.isEmpty()) + { + new RDBMSBackendAssessor(qInstanceAssessor, this, tables).assess(); + } + } }