diff --git a/pom.xml b/pom.xml index 180d2ad4..82ca2e0d 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,11 @@ json 20210307 + + org.apache.commons + commons-csv + 1.8 + @@ -122,6 +127,14 @@ + github @@ -129,6 +142,5 @@ https://maven.pkg.github.com/Kingsrook/qqq-maven-registry ---> diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/MetaDataAction.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/MetaDataAction.java new file mode 100644 index 00000000..f0071d8f --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/MetaDataAction.java @@ -0,0 +1,37 @@ +package com.kingsrook.qqq.backend.core.actions; + + +import java.util.LinkedHashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.MetaDataRequest; +import com.kingsrook.qqq.backend.core.model.actions.MetaDataResult; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MetaDataAction +{ + /******************************************************************************* + ** + *******************************************************************************/ + public MetaDataResult execute(MetaDataRequest metaDataRequest) throws QException + { + // todo pre-customization - just get to modify the request? + MetaDataResult metaDataResult = new MetaDataResult(); + + Map tables = new LinkedHashMap<>(); + for(Map.Entry entry : metaDataRequest.getInstance().getTables().entrySet()) + { + tables.put(entry.getKey(), new QFrontendTableMetaData(entry.getValue(), false)); + } + + metaDataResult.setTables(tables); + // todo post-customization - can do whatever w/ the result if you want + + return metaDataResult; + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java new file mode 100644 index 00000000..1e30f980 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -0,0 +1,122 @@ +package com.kingsrook.qqq.backend.core.adapters; + + +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.actions.AbstractQFieldMapping; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class CsvToQRecordAdapter +{ + + /******************************************************************************* + ** todo - meta-data validation, mapping, type handling + *******************************************************************************/ + public List buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping) + { + if(!StringUtils.hasContent(csv)) + { + throw (new IllegalArgumentException("Empty csv value was provided.")); + } + + List rs = new ArrayList<>(); + try + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there's no mapping (e.g., table-standard field names), or key-based mapping, then first row is headers // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(mapping == null || AbstractQFieldMapping.SourceType.KEY.equals(mapping.getSourceType())) + { + CSVParser csvParser = new CSVParser(new StringReader(csv), + CSVFormat.DEFAULT + .withFirstRecordAsHeader() + .withIgnoreHeaderCase() + .withTrim()); + + List headers = csvParser.getHeaderNames(); + List csvRecords = csvParser.getRecords(); + for(CSVRecord csvRecord : csvRecords) + { + ////////////////////////////////////////////////////////////////// + // put values from the CSV record into a map of header -> value // + ////////////////////////////////////////////////////////////////// + Map csvValues = new HashMap<>(); + for(String header : headers) + { + csvValues.put(header, csvRecord.get(header)); + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + QRecord qRecord = new QRecord(); + rs.add(qRecord); + for(QFieldMetaData field : table.getFields().values()) + { + String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName())); + qRecord.setValue(field.getName(), csvValues.get(fieldSource)); + } + } + } + else if(AbstractQFieldMapping.SourceType.INDEX.equals(mapping.getSourceType())) + { + /////////////////////////////// + // else, index-based mapping // + /////////////////////////////// + CSVParser csvParser = new CSVParser(new StringReader(csv), + CSVFormat.DEFAULT + .withTrim()); + + List csvRecords = csvParser.getRecords(); + for(CSVRecord csvRecord : csvRecords) + { + ///////////////////////////////////////////////////////////////// + // put values from the CSV record into a map of index -> value // + ///////////////////////////////////////////////////////////////// + Map csvValues = new HashMap<>(); + int index = 1; + for(String value : csvRecord) + { + csvValues.put(index++, value); + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + QRecord qRecord = new QRecord(); + rs.add(qRecord); + for(QFieldMetaData field : table.getFields().values()) + { + Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName()); + qRecord.setValue(field.getName(), csvValues.get(fieldIndex)); + } + } + } + else + { + throw (new IllegalArgumentException("Unrecognized mapping source type: " + mapping.getSourceType())); + } + } + catch(IOException e) + { + throw (new IllegalArgumentException("Error parsing CSV: " + e.getMessage(), e)); + } + + return (rs); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java b/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java index bf54b9af..eb922b84 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java @@ -2,6 +2,7 @@ package com.kingsrook.qqq.backend.core.adapters; import com.kingsrook.qqq.backend.core.model.actions.AbstractQFieldMapping; +import com.kingsrook.qqq.backend.core.model.actions.QIndexBasedFieldMapping; import com.kingsrook.qqq.backend.core.model.actions.QKeyBasedFieldMapping; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -33,12 +34,37 @@ public class JsonToQFieldMappingAdapter // look at the keys in the mapping - if they're strings, then we're doing key-based mapping // // if they're numbers, then we're doing index based -- and if they're a mix, that's illegal // ////////////////////////////////////////////////////////////////////////////////////////////// + AbstractQFieldMapping.SourceType sourceType = determineSourceType(jsonObject); - QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping(); - for(String key : jsonObject.keySet()) + @SuppressWarnings("rawtypes") + AbstractQFieldMapping mapping = null; + + switch(sourceType) { - mapping.addMapping(key, jsonObject.getString(key)); + case KEY: + { + mapping = new QKeyBasedFieldMapping(); + for(String fieldName : jsonObject.keySet()) + { + ((QKeyBasedFieldMapping) mapping).addMapping(fieldName, jsonObject.getString(fieldName)); + } + break; + } + case INDEX: + { + mapping = new QIndexBasedFieldMapping(); + for(String fieldName : jsonObject.keySet()) + { + ((QIndexBasedFieldMapping) mapping).addMapping(fieldName, jsonObject.getInt(fieldName)); + } + break; + } + default: + { + throw (new IllegalArgumentException("Unsupported sourceType: " + sourceType)); + } } + return (mapping); } catch(JSONException je) @@ -47,4 +73,30 @@ public class JsonToQFieldMappingAdapter } } + + + /******************************************************************************* + ** + *******************************************************************************/ + private AbstractQFieldMapping.SourceType determineSourceType(JSONObject jsonObject) + { + for(String fieldName : jsonObject.keySet()) + { + Object sourceObject = jsonObject.get(fieldName); + if(sourceObject instanceof String) + { + return (AbstractQFieldMapping.SourceType.KEY); + } + else if(sourceObject instanceof Integer) + { + return (AbstractQFieldMapping.SourceType.INDEX); + } + else + { + throw new IllegalArgumentException("Source object is unsupported type: " + sourceObject.getClass().getSimpleName()); + } + } + throw new IllegalArgumentException("No fields were found in the mapping."); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapter.java b/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapter.java index d4fdb33e..cef0959f 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapter.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapter.java @@ -4,7 +4,9 @@ package com.kingsrook.qqq.backend.core.adapters; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.AbstractQFieldMapping; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.json.JSONArray; @@ -21,7 +23,7 @@ public class JsonToQRecordAdapter /******************************************************************************* ** todo - meta-data validation, mapping, type handling *******************************************************************************/ - public List buildRecordsFromJson(String json) + public List buildRecordsFromJson(String json, QTableMetaData table, AbstractQFieldMapping mapping) { if(!StringUtils.hasContent(json)) { diff --git a/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QInstanceValidationException.java b/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QInstanceValidationException.java new file mode 100644 index 00000000..223643e3 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QInstanceValidationException.java @@ -0,0 +1,82 @@ +package com.kingsrook.qqq.backend.core.exceptions; + + +import java.util.Arrays; +import java.util.List; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QInstanceValidationException extends QException +{ + private List reasons; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QInstanceValidationException(String message) + { + super(message); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QInstanceValidationException(List reasons) + { + super( + (reasons != null && reasons.size() > 0) + ? "Instance validation failed for the following reasons: " + StringUtils.joinWithCommasAndAnd(reasons) + : "Validation failed, but no reasons were provided"); + + if(reasons != null && reasons.size() > 0) + { + this.reasons = reasons; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QInstanceValidationException(String... reasons) + { + super( + (reasons != null && reasons.length > 0) + ? "Instance validation failed for the following reasons: " + StringUtils.joinWithCommasAndAnd(Arrays.stream(reasons).toList()) + : "Validation failed, but no reasons were provided"); + + if(reasons != null && reasons.length > 0) + { + this.reasons = Arrays.stream(reasons).toList(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QInstanceValidationException(String message, Throwable cause) + { + super(message, cause); + } + + + + /******************************************************************************* + ** Getter for reasons + ** + *******************************************************************************/ + public List getReasons() + { + return reasons; + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java new file mode 100644 index 00000000..20d54a95 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -0,0 +1,73 @@ +package com.kingsrook.qqq.backend.core.instances; + + +import java.util.Locale; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QInstanceEnricher +{ + /******************************************************************************* + ** + *******************************************************************************/ + public void enrich(QInstance qInstance) + { + if (qInstance.getTables() != null) + { + qInstance.getTables().values().forEach(this::enrich); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void enrich(QTableMetaData table) + { + if(!StringUtils.hasContent(table.getLabel())) + { + table.setLabel(nameToLabel(table.getName())); + } + + if (table.getFields() != null) + { + table.getFields().values().forEach(this::enrich); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void enrich(QFieldMetaData field) + { + if(!StringUtils.hasContent(field.getLabel())) + { + field.setLabel(nameToLabel(field.getName())); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String nameToLabel(String name) + { + if(name == null) + { + return (null); + } + + return (name.substring(0, 1).toUpperCase(Locale.ROOT) + name.substring(1).replaceAll("([A-Z])", " $1")); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidationKey.java b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidationKey.java new file mode 100644 index 00000000..9e11fbe8 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidationKey.java @@ -0,0 +1,18 @@ +package com.kingsrook.qqq.backend.core.instances; + + +/******************************************************************************* + ** + *******************************************************************************/ +public final class QInstanceValidationKey +{ + /******************************************************************************* + ** package-private constructor, so that only this package can create an instance + ** of this class, but an instance of this class is required to mark an instance + ** as validated, so no one can cheat. + *******************************************************************************/ + QInstanceValidationKey() + { + + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java new file mode 100644 index 00000000..662f9415 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -0,0 +1,104 @@ +package com.kingsrook.qqq.backend.core.instances; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QInstanceValidator +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public void validate(QInstance qInstance) throws QInstanceValidationException + { + if(qInstance.getHasBeenValidated()) + { + ////////////////////////////////////////// + // don't re-validate if previously done // + ////////////////////////////////////////// + return; + } + + try + { + ///////////////////////////////////////////////////////////////////////////////////////////////// + // before validation, enrich the object (e.g., to fill in values that the user doesn't have to // + ///////////////////////////////////////////////////////////////////////////////////////////////// + new QInstanceEnricher().enrich(qInstance); + } + catch(Exception e) + { + e.printStackTrace(); + throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e)); + } + + ////////////////////////////////////////////////////////////////////////// + // do the validation checks - a good qInstance has all conditions TRUE! // + ////////////////////////////////////////////////////////////////////////// + List errors = new ArrayList<>(); + try + { + if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getBackends()), "At least 1 backend must be defined.")) + { + qInstance.getBackends().forEach((backendName, backend) -> + { + assertCondition(errors, Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); + }); + } + + if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined.")) + { + qInstance.getTables().forEach((tableName, table) -> + { + assertCondition(errors, Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + "."); + + if(assertCondition(errors, StringUtils.hasContent(table.getBackendName()), "Missing backend name for table " + tableName + ".")) + { + if(CollectionUtils.nullSafeHasContents(qInstance.getBackends())) + { + assertCondition(errors, qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + "."); + } + } + }); + } + } + catch(Exception e) + { + e.printStackTrace(); + throw (new QInstanceValidationException("Error performing qInstance validation.", e)); + } + + if(!errors.isEmpty()) + { + throw (new QInstanceValidationException(errors)); + } + + qInstance.setHasBeenValidated(new QInstanceValidationKey()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean assertCondition(List errors, boolean condition, String message) + { + if(!condition) + { + errors.add(message); + } + + return (condition); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQFieldMapping.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQFieldMapping.java index f5c6cb89..fa1bf8b8 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQFieldMapping.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQFieldMapping.java @@ -2,12 +2,56 @@ package com.kingsrook.qqq.backend.core.model.actions; /******************************************************************************* + ** For bulk-loads, define where a QField comes from in an input data source. ** *******************************************************************************/ public abstract class AbstractQFieldMapping { + /******************************************************************************* - ** Returns the fieldName for the input 'key' or 'index'. + ** Enum to define the types of sources a mapping may use *******************************************************************************/ - public abstract String getMappedField(T t); + @SuppressWarnings("rawtypes") + public enum SourceType + { + KEY(String.class), + INDEX(Integer.class); + + private Class sourceClass; + + + + /******************************************************************************* + ** enum constructor + *******************************************************************************/ + SourceType(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + } + + + + /******************************************************************************* + ** For a given field, return its source - a key (e.g., from a json object or csv + ** with a header row) or an index (for a csv w/o a header) + *******************************************************************************/ + public abstract T getFieldSource(String fieldName); + + + /******************************************************************************* + ** for a mapping instance, get what its source-type is + *******************************************************************************/ + public abstract SourceType getSourceType(); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java index 4481202c..3a0d5c57 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java @@ -1,8 +1,9 @@ package com.kingsrook.qqq.backend.core.model.actions; -import com.kingsrook.qqq.backend.core.model.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; /******************************************************************************* @@ -15,13 +16,6 @@ public abstract class AbstractQRequest - /******************************************************************************* - ** - *******************************************************************************/ - public abstract QBackendMetaData getBackend(); - - - /******************************************************************************* ** *******************************************************************************/ @@ -37,6 +31,23 @@ public abstract class AbstractQRequest public AbstractQRequest(QInstance instance) { this.instance = instance; + + //////////////////////////////////////////////////////////// + // if this instance hasn't been validated yet, do so now // + // noting that this will also enrich any missing metaData // + //////////////////////////////////////////////////////////// + if(! instance.getHasBeenValidated()) + { + try + { + new QInstanceValidator().validate(instance); + } + catch(QInstanceValidationException e) + { + System.err.println(e.getMessage()); + throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage())); + } + } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQTableRequest.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQTableRequest.java index 2bb30f0a..55fedaf6 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQTableRequest.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQTableRequest.java @@ -1,8 +1,8 @@ package com.kingsrook.qqq.backend.core.model.actions; -import com.kingsrook.qqq.backend.core.model.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; @@ -18,13 +18,13 @@ public abstract class AbstractQTableRequest extends AbstractQRequest /******************************************************************************* ** *******************************************************************************/ - @Override public QBackendMetaData getBackend() { return (instance.getBackendForTable(getTableName())); } + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/InsertRequest.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/InsertRequest.java index 68d2ebde..81fe07f6 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/InsertRequest.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/InsertRequest.java @@ -2,8 +2,8 @@ package com.kingsrook.qqq.backend.core.model.actions; import java.util.List; -import com.kingsrook.qqq.backend.core.model.QInstance; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; /******************************************************************************* diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/MetaDataRequest.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/MetaDataRequest.java new file mode 100644 index 00000000..4bbe2638 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/MetaDataRequest.java @@ -0,0 +1,30 @@ +package com.kingsrook.qqq.backend.core.model.actions; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MetaDataRequest extends AbstractQRequest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public MetaDataRequest() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MetaDataRequest(QInstance instance) + { + super(instance); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/MetaDataResult.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/MetaDataResult.java new file mode 100644 index 00000000..db73bddf --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/MetaDataResult.java @@ -0,0 +1,37 @@ +package com.kingsrook.qqq.backend.core.model.actions; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData; + + +/******************************************************************************* + * Result for a metaData action + * + *******************************************************************************/ +public class MetaDataResult extends AbstractQResult +{ + Map tables; + + + + /******************************************************************************* + ** Getter for tables + ** + *******************************************************************************/ + public Map getTables() + { + return tables; + } + + + + /******************************************************************************* + ** Setter for tables + ** + *******************************************************************************/ + public void setTables(Map tables) + { + this.tables = tables; + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QIndexBasedFieldMapping.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QIndexBasedFieldMapping.java index d68c31c9..bbda0c31 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QIndexBasedFieldMapping.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QIndexBasedFieldMapping.java @@ -10,7 +10,7 @@ import java.util.Map; *******************************************************************************/ public class QIndexBasedFieldMapping extends AbstractQFieldMapping { - private Map mapping; + private Map mapping; @@ -18,14 +18,14 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping ** *******************************************************************************/ @Override - public String getMappedField(Integer key) + public Integer getFieldSource(String fieldName) { if(mapping == null) { return (null); } - return (mapping.get(key)); + return (mapping.get(fieldName)); } @@ -33,13 +33,24 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping /******************************************************************************* ** *******************************************************************************/ - public void addMapping(Integer key, String fieldName) + @Override + public SourceType getSourceType() + { + return (SourceType.INDEX); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addMapping(String fieldName, Integer key) { if(mapping == null) { mapping = new LinkedHashMap<>(); } - mapping.put(key, fieldName); + mapping.put(fieldName, key); } @@ -47,9 +58,9 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping /******************************************************************************* ** *******************************************************************************/ - public QIndexBasedFieldMapping withMapping(Integer key, String fieldName) + public QIndexBasedFieldMapping withMapping(String fieldName, Integer key) { - addMapping(key, fieldName); + addMapping(fieldName, key); return (this); } @@ -59,7 +70,7 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping ** Getter for mapping ** *******************************************************************************/ - public Map getMapping() + public Map getMapping() { return mapping; } @@ -70,7 +81,7 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping ** Setter for mapping ** *******************************************************************************/ - public void setMapping(Map mapping) + public void setMapping(Map mapping) { this.mapping = mapping; } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QKeyBasedFieldMapping.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QKeyBasedFieldMapping.java index 08a1cd11..82b644a4 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QKeyBasedFieldMapping.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QKeyBasedFieldMapping.java @@ -18,14 +18,14 @@ public class QKeyBasedFieldMapping extends AbstractQFieldMapping ** *******************************************************************************/ @Override - public String getMappedField(String key) + public String getFieldSource(String fieldName) { if(mapping == null) { return (null); } - return (mapping.get(key)); + return (mapping.get(fieldName)); } @@ -33,13 +33,24 @@ public class QKeyBasedFieldMapping extends AbstractQFieldMapping /******************************************************************************* ** *******************************************************************************/ - public void addMapping(String key, String fieldName) + @Override + public SourceType getSourceType() + { + return (SourceType.KEY); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addMapping(String fieldName, String key) { if(mapping == null) { mapping = new LinkedHashMap<>(); } - mapping.put(key, fieldName); + mapping.put(fieldName, key); } @@ -47,9 +58,9 @@ public class QKeyBasedFieldMapping extends AbstractQFieldMapping /******************************************************************************* ** *******************************************************************************/ - public QKeyBasedFieldMapping withMapping(String key, String fieldName) + public QKeyBasedFieldMapping withMapping(String fieldName, String key) { - addMapping(key, fieldName); + addMapping(fieldName, key); return (this); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QueryRequest.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QueryRequest.java index ba871fdf..863f721c 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QueryRequest.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/QueryRequest.java @@ -1,7 +1,7 @@ package com.kingsrook.qqq.backend.core.model.actions; -import com.kingsrook.qqq.backend.core.model.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; /******************************************************************************* diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index a4d14204..939deade 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -3,6 +3,7 @@ package com.kingsrook.qqq.backend.core.model.metadata; import java.util.HashMap; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonFilter; /******************************************************************************* @@ -12,6 +13,8 @@ public class QBackendMetaData { private String name; private String type; + + @JsonFilter("secretsFilter") private Map values; diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/QInstance.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java similarity index 71% rename from src/main/java/com/kingsrook/qqq/backend/core/model/QInstance.java rename to src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 70c219da..48af755a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/QInstance.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -1,10 +1,10 @@ -package com.kingsrook.qqq.backend.core.model; +package com.kingsrook.qqq.backend.core.model.metadata; import java.util.HashMap; import java.util.Map; -import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey; /******************************************************************************* @@ -12,9 +12,18 @@ import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; *******************************************************************************/ public class QInstance { + /////////////////////////////////////////////////////////////////////////////// + // Do not let the backend data be serialized - e.g., sent to a frontend user // + /////////////////////////////////////////////////////////////////////////////// + @JsonIgnore private Map backends = new HashMap<>(); + private Map tables = new HashMap<>(); + // todo - lock down the object (no more changes allowed) after it's been validated? + @JsonIgnore + private boolean hasBeenValidated = false; + /******************************************************************************* @@ -29,16 +38,27 @@ public class QInstance } QBackendMetaData backend = backends.get(table.getBackendName()); - if(backend == null) - { - throw (new IllegalArgumentException("Table [" + tableName + "] specified a backend name [" + table.getBackendName() + "] which was found in this instance.")); - } + + ////////////////////////////////////////////////////////////////////////////////////////////// + // validation should already let us know that this is valid, so no need to check/throw here // + ////////////////////////////////////////////////////////////////////////////////////////////// return (backend); } + /******************************************************************************* + ** Setter for hasBeenValidated + ** + *******************************************************************************/ + public void setHasBeenValidated(QInstanceValidationKey key) + { + this.hasBeenValidated = true; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -140,4 +160,16 @@ public class QInstance { this.tables = tables; } + + + + /******************************************************************************* + ** Getter for hasBeenValidated + ** + *******************************************************************************/ + public boolean getHasBeenValidated() + { + return hasBeenValidated; + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java index ed3540aa..bcecd75f 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java @@ -1,9 +1,7 @@ package com.kingsrook.qqq.backend.core.model.metadata; -import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; @@ -13,10 +11,10 @@ import java.util.Map; public class QTableMetaData { private String name; + private String label; private String backendName; private String primaryKeyField; - private List fields; - private Map _fieldMap; + private Map fields; @@ -30,7 +28,7 @@ public class QTableMetaData throw (new IllegalArgumentException("Table [" + name + "] does not have its fields defined.")); } - QFieldMetaData field = getFieldMap().get(fieldName); + QFieldMetaData field = getFields().get(fieldName); if(field == null) { throw (new IllegalArgumentException("Field [" + fieldName + "] was not found in table [" + name + "].")); @@ -41,25 +39,6 @@ public class QTableMetaData - /******************************************************************************* - ** - *******************************************************************************/ - private Map getFieldMap() - { - if(_fieldMap == null) - { - _fieldMap = new LinkedHashMap<>(); - for(QFieldMetaData field : fields) - { - _fieldMap.put(field.getName(), field); - } - } - - return (_fieldMap); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -91,6 +70,37 @@ public class QTableMetaData + /******************************************************************************* + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData withLabel(String label) + { + this.label = label; + return (this); + } + + + /******************************************************************************* ** Getter for backendName ** @@ -158,7 +168,7 @@ public class QTableMetaData /******************************************************************************* ** *******************************************************************************/ - public List getFields() + public Map getFields() { return fields; } @@ -166,9 +176,10 @@ public class QTableMetaData /******************************************************************************* + ** Setter for fields ** *******************************************************************************/ - public void setFields(List fields) + public void setFields(Map fields) { this.fields = fields; } @@ -178,7 +189,7 @@ public class QTableMetaData /******************************************************************************* ** *******************************************************************************/ - public QTableMetaData withFields(List fields) + public QTableMetaData withFields(Map fields) { this.fields = fields; return (this); @@ -193,9 +204,9 @@ public class QTableMetaData { if(this.fields == null) { - this.fields = new ArrayList<>(); + this.fields = new LinkedHashMap<>(); } - this.fields.add(field); + this.fields.put(field.getName(), field); } @@ -207,9 +218,9 @@ public class QTableMetaData { if(this.fields == null) { - this.fields = new ArrayList<>(); + this.fields = new LinkedHashMap<>(); } - this.fields.add(field); + this.fields.put(field.getName(), field); return (this); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java new file mode 100644 index 00000000..d11c66f5 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java @@ -0,0 +1,69 @@ +package com.kingsrook.qqq.backend.core.model.metadata.frontend; + + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; + + +/******************************************************************************* + * Version of QTableMetaData that's meant for transmitting to a frontend. + * e.g., it excludes backend-only details. + *******************************************************************************/ +@JsonInclude(Include.NON_NULL) +public class QFrontendFieldMetaData +{ + private String name; + private String label; + private QFieldType type; + + ////////////////////////////////////////////////////////////////////////////////// + // do not add setters. take values from the source-object in the constructor!! // + ////////////////////////////////////////////////////////////////////////////////// + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFrontendFieldMetaData(QFieldMetaData fieldMetaData) + { + this.name = fieldMetaData.getName(); + this.label = fieldMetaData.getLabel(); + this.type = fieldMetaData.getType(); + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public QFieldType getType() + { + return type; + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java new file mode 100644 index 00000000..9c0a9c3b --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -0,0 +1,92 @@ +package com.kingsrook.qqq.backend.core.model.metadata.frontend; + + +import java.util.HashMap; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; + + +/******************************************************************************* + * Version of QTableMetaData that's meant for transmitting to a frontend. + * e.g., it excludes backend-only details. + *******************************************************************************/ +@JsonInclude(Include.NON_NULL) +public class QFrontendTableMetaData +{ + private String name; + private String label; + private String primaryKeyField; + private Map fields; + + ////////////////////////////////////////////////////////////////////////////////// + // do not add setters. take values from the source-object in the constructor!! // + ////////////////////////////////////////////////////////////////////////////////// + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFrontendTableMetaData(QTableMetaData tableMetaData, boolean includeFields) + { + this.name = tableMetaData.getName(); + this.label = tableMetaData.getLabel(); + + if(includeFields) + { + this.primaryKeyField = tableMetaData.getPrimaryKeyField(); + this.fields = new HashMap<>(); + for(Map.Entry entry : tableMetaData.getFields().entrySet()) + { + this.fields.put(entry.getKey(), new QFrontendFieldMetaData(entry.getValue())); + } + } + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Getter for primaryKeyField + ** + *******************************************************************************/ + public String getPrimaryKeyField() + { + return primaryKeyField; + } + + + + /******************************************************************************* + ** Getter for fields + ** + *******************************************************************************/ + public Map getFields() + { + return fields; + } +} \ No newline at end of file diff --git a/src/main/java/com/kingsrook/qqq/backend/core/modules/QModuleDispatcher.java b/src/main/java/com/kingsrook/qqq/backend/core/modules/QModuleDispatcher.java index 774ebb69..98e61b12 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/modules/QModuleDispatcher.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/modules/QModuleDispatcher.java @@ -52,7 +52,7 @@ public class QModuleDispatcher } catch(Exception e) { - throw (new QModuleDispatchException("Error getting module", e)); + throw (new QModuleDispatchException("Error getting q backend module of type: " + backend.getType(), e)); } } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java b/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java index 67e54e04..a4d41eae 100755 --- a/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java @@ -30,6 +30,22 @@ public class CollectionUtils + /******************************************************************************* + ** true if c is null or it's empty + ** + *******************************************************************************/ + public static boolean nullSafeIsEmpty(Map c) + { + if(c == null || c.isEmpty()) + { + return (true); + } + + return (false); + } + + + /******************************************************************************* ** true if c is NOT null and it's not empty ** @@ -41,6 +57,17 @@ public class CollectionUtils + /******************************************************************************* + ** true if c is NOT null and it's not empty + ** + *******************************************************************************/ + public static boolean nullSafeHasContents(Map c) + { + return (!nullSafeIsEmpty(c)); + } + + + /******************************************************************************* ** 0 if c is empty, otherwise, its size. ** @@ -57,6 +84,22 @@ public class CollectionUtils + /******************************************************************************* + ** 0 if c is empty, otherwise, its size. + ** + *******************************************************************************/ + public static int nullSafeSize(Map c) + { + if(c == null) + { + return (0); + } + + return (c.size()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java b/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java index ec386b8a..b2dd056a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java @@ -94,6 +94,15 @@ public class JsonUtils ObjectMapper mapper = new ObjectMapper() .registerModule(new JavaTimeModule()) .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + + /* todo - some future version we may need to do inclusion/exclusion lists like this: + // this is what we'd put on the class or member we wanted to 'filter': @JsonFilter("secretsFilter") + + SimpleFilterProvider filterProvider = new SimpleFilterProvider(); + filterProvider.addFilter("secretsFilter", SimpleBeanPropertyFilter.serializeAllExcept("password")); + mapper.setFilterProvider(filterProvider); + */ + return (mapper); } diff --git a/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java b/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java new file mode 100644 index 00000000..acba0fb1 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java @@ -0,0 +1,222 @@ +package com.kingsrook.qqq.backend.core.adapters; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.QIndexBasedFieldMapping; +import com.kingsrook.qqq.backend.core.model.actions.QKeyBasedFieldMapping; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** + *******************************************************************************/ +class CsvToQRecordAdapterTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_nullInput() + { + testExpectedToThrow(null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_emptyStringInput() + { + testExpectedToThrow(""); + } + + /******************************************************************************* + ** + *******************************************************************************/ + /* todo? + @Test + public void test_buildRecordsFromCsv_inputDoesntLookLikeCsv() + { + testExpectedToThrow("CSV"); + } + */ + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testExpectedToThrow(String csv) + { + try + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csv, TestUtils.defineTablePerson(), null); + System.out.println(qRecords); + } + catch(IllegalArgumentException iae) + { + System.out.println("Threw expected exception"); + return; + } + + fail("Didn't throw expected exception"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_emptyList() + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader(), TestUtils.defineTablePerson(), null); + assertNotNull(qRecords); + assertTrue(qRecords.isEmpty()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getPersonCsvHeader() + { + return (""" + "id","createDate","modifyDate","firstName","lastName","birthDate","email"\r + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getPersonCsvRow2() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"\r + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getPersonCsvRow1() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"\r + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_oneRowStandardHeaderNoMapping() + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1(), TestUtils.defineTablePerson(), null); + assertNotNull(qRecords); + assertEquals(1, qRecords.size()); + QRecord qRecord = qRecords.get(0); + assertEquals("John", qRecord.getValue("firstName")); + assertEquals("1980-01-01", qRecord.getValue("birthDate")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_twoRowsStandardHeaderNoMapping() + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), null); + assertNotNull(qRecords); + assertEquals(2, qRecords.size()); + QRecord qRecord1 = qRecords.get(0); + assertEquals("John", qRecord1.getValue("firstName")); + assertEquals("1980-01-01", qRecord1.getValue("birthDate")); + QRecord qRecord2 = qRecords.get(1); + assertEquals("Jane", qRecord2.getValue("firstName")); + assertEquals("1981-01-01", qRecord2.getValue("birthDate")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_oneRowCustomKeyBasedMapping() + { + String csvCustomHeader = """ + "id","created","modified","first","last","birthday","email"\r + """; + + QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping() + .withMapping("id", "id") + .withMapping("createDate", "created") + .withMapping("modifyDate", "modified") + .withMapping("firstName", "first") + .withMapping("lastName", "last") + .withMapping("birthDate", "birthday") + .withMapping("email", "email"); + + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csvCustomHeader + getPersonCsvRow1(), TestUtils.defineTablePerson(), mapping); + assertNotNull(qRecords); + assertEquals(1, qRecords.size()); + QRecord qRecord = qRecords.get(0); + assertEquals("John", qRecord.getValue("firstName")); + assertEquals("1980-01-01", qRecord.getValue("birthDate")); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_twoRowsCustomIndexBasedMapping() + { + int index = 1; + QIndexBasedFieldMapping mapping = new QIndexBasedFieldMapping() + .withMapping("id", index++) + .withMapping("createDate", index++) + .withMapping("modifyDate", index++) + .withMapping("firstName", index++) + .withMapping("lastName", index++) + .withMapping("birthDate", index++) + .withMapping("email", index++); + + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), mapping); + assertNotNull(qRecords); + assertEquals(2, qRecords.size()); + QRecord qRecord1 = qRecords.get(0); + assertEquals("John", qRecord1.getValue("firstName")); + assertEquals("1980-01-01", qRecord1.getValue("birthDate")); + QRecord qRecord2 = qRecords.get(1); + assertEquals("Jane", qRecord2.getValue("firstName")); + assertEquals("1981-01-01", qRecord2.getValue("birthDate")); + } + + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java b/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java index 1bd256dd..425ca107 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java @@ -2,6 +2,7 @@ package com.kingsrook.qqq.backend.core.adapters; import com.kingsrook.qqq.backend.core.model.actions.AbstractQFieldMapping; +import com.kingsrook.qqq.backend.core.model.actions.QIndexBasedFieldMapping; import com.kingsrook.qqq.backend.core.model.actions.QKeyBasedFieldMapping; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -51,7 +52,7 @@ class JsonToQFieldMappingAdapterTest ** *******************************************************************************/ @Test - public void test_buildMappingFromJson_validInput() + public void test_buildMappingFromJson_validKeyBasedInput() { JsonToQFieldMappingAdapter jsonToQFieldMappingAdapter = new JsonToQFieldMappingAdapter(); AbstractQFieldMapping mapping = (QKeyBasedFieldMapping) jsonToQFieldMappingAdapter.buildMappingFromJson(""" @@ -63,9 +64,67 @@ class JsonToQFieldMappingAdapterTest System.out.println(mapping); assertNotNull(mapping); - // todo - are we backwards here?? - assertEquals("source1", mapping.getMappedField("Field1")); - assertEquals("source2", mapping.getMappedField("Field2")); + assertEquals("source1", mapping.getFieldSource("Field1")); + assertEquals("source2", mapping.getFieldSource("Field2")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildMappingFromJson_validIndexBasedInput() + { + JsonToQFieldMappingAdapter jsonToQFieldMappingAdapter = new JsonToQFieldMappingAdapter(); + AbstractQFieldMapping mapping = (QIndexBasedFieldMapping) jsonToQFieldMappingAdapter.buildMappingFromJson(""" + { + "Field1": 1, + "Field2": 2, + } + """); + System.out.println(mapping); + assertNotNull(mapping); + + assertEquals(1, mapping.getFieldSource("Field1")); + assertEquals(2, mapping.getFieldSource("Field2")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildMappingFromJson_unsupportedTypeForSource() + { + testExpectedToThrow(""" + { + "Field1": [1, 2], + "Field2": {"A": "B"} + } + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildMappingFromJson_emptyMapping() + { + testExpectedToThrow("{}"); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildMappingFromJson_inputJsonList() + { + testExpectedToThrow("[]"); } diff --git a/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapterTest.java b/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapterTest.java index 21923faa..2882391f 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapterTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapterTest.java @@ -3,6 +3,7 @@ package com.kingsrook.qqq.backend.core.adapters; import java.util.List; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -65,7 +66,7 @@ class JsonToQRecordAdapterTest try { JsonToQRecordAdapter jsonToQRecordAdapter = new JsonToQRecordAdapter(); - List qRecords = jsonToQRecordAdapter.buildRecordsFromJson(json); + List qRecords = jsonToQRecordAdapter.buildRecordsFromJson(json, TestUtils.defineTablePerson(), null); System.out.println(qRecords); } catch(IllegalArgumentException iae) @@ -86,7 +87,7 @@ class JsonToQRecordAdapterTest public void test_buildRecordsFromJson_emptyList() { JsonToQRecordAdapter jsonToQRecordAdapter = new JsonToQRecordAdapter(); - List qRecords = jsonToQRecordAdapter.buildRecordsFromJson("[]"); + List qRecords = jsonToQRecordAdapter.buildRecordsFromJson("[]", TestUtils.defineTablePerson(), null); assertNotNull(qRecords); assertTrue(qRecords.isEmpty()); } @@ -105,7 +106,7 @@ class JsonToQRecordAdapterTest "field1":"value1", "field2":"value2" } - """); + """, TestUtils.defineTablePerson(), null); assertNotNull(qRecords); assertEquals(1, qRecords.size()); assertEquals("value1", qRecords.get(0).getValue("field1")); @@ -126,7 +127,7 @@ class JsonToQRecordAdapterTest { "field1":"value1", "field2":"value2" }, { "fieldA":"valueA", "fieldB":"valueB" } ] - """); + """, TestUtils.defineTablePerson(), null); assertNotNull(qRecords); assertEquals(2, qRecords.size()); assertEquals("value1", qRecords.get(0).getValue("field1")); diff --git a/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java new file mode 100644 index 00000000..5ce4a536 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -0,0 +1,188 @@ +package com.kingsrook.qqq.backend.core.instances; + + +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** + *******************************************************************************/ +class QInstanceValidatorTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_validatePass() throws QInstanceValidationException + { + new QInstanceValidator().validate(TestUtils.defineInstance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_validateNullBackends() + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.setBackends(null); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + assertReason("At least 1 backend must be defined", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_validateEmptyBackends() + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.setBackends(new HashMap<>()); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + assertReason("At least 1 backend must be defined", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_validateNullTables() + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.setTables(null); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + assertReason("At least 1 table must be defined", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_validateEmptyTables() + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.setTables(new HashMap<>()); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + assertReason("At least 1 table must be defined", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_validateInconsistentNames() + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable("person").setName("notPerson"); + qInstance.getBackend("default").setName("notDefault"); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + assertReason("Inconsistent naming for table", e); + assertReason("Inconsistent naming for backend", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_validateTableWithoutBackend() + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable("person").setBackendName(null); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + assertReason("Missing backend name for table", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_validateTableWithMissingBackend() + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.getTable("person").setBackendName("notARealBackend"); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + assertReason("Unrecognized backend", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void assertReason(String reason, QInstanceValidationException e) + { + assertNotNull(e.getReasons()); + assertTrue(e.getReasons().stream().anyMatch(s -> s.contains(reason))); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java new file mode 100644 index 00000000..ebfb0492 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -0,0 +1,65 @@ +package com.kingsrook.qqq.backend.core.utils; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestUtils +{ + /******************************************************************************* + ** + *******************************************************************************/ + public static QInstance defineInstance() + { + QInstance qInstance = new QInstance(); + qInstance.addBackend(defineBackend()); + qInstance.addTable(defineTablePerson()); + return (qInstance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QBackendMetaData defineBackend() + { + return new QBackendMetaData() + .withName("default") + .withType("rdbms") + .withValue("vendor", "h2") + .withValue("hostName", "mem") + .withValue("databaseName", "test_database") + .withValue("username", "sa") + .withValue("password", ""); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineTablePerson() + { + return new QTableMetaData() + .withName("person") + .withLabel("Person") + .withBackendName(defineBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("firstName", QFieldType.STRING)) + .withField(new QFieldMetaData("lastName", QFieldType.STRING)) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE)) + .withField(new QFieldMetaData("email", QFieldType.STRING)); + } + +}