mirror of
https://github.com/Kingsrook/qqq-frontend-material-dashboard.git
synced 2025-07-19 05:40:44 +00:00
CE-1955 - Initial checkin of qfmd support for bulk-load
This commit is contained in:
600
src/qqq/models/processes/BulkLoadModels.ts
Normal file
600
src/qqq/models/processes/BulkLoadModels.ts
Normal file
@ -0,0 +1,600 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2024. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
import {QFieldMetaData} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QFieldMetaData";
|
||||
import {QRecord} from "@kingsrook/qqq-frontend-core/lib/model/QRecord";
|
||||
|
||||
export type ValueType = "defaultValue" | "column";
|
||||
|
||||
/***************************************************************************
|
||||
** model of a single field that's part of a bulk-load profile/mapping
|
||||
***************************************************************************/
|
||||
export class BulkLoadField
|
||||
{
|
||||
field: QFieldMetaData;
|
||||
tableStructure: BulkLoadTableStructure;
|
||||
|
||||
valueType: ValueType;
|
||||
columnIndex?: number;
|
||||
headerName?: string = null;
|
||||
defaultValue?: any = null;
|
||||
doValueMapping: boolean = false;
|
||||
|
||||
wideLayoutIndexPath: number[] = [];
|
||||
|
||||
error: string = null;
|
||||
|
||||
key: string;
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
constructor(field: QFieldMetaData, tableStructure: BulkLoadTableStructure, valueType: ValueType = "column", columnIndex?: number, headerName?: string, defaultValue?: any, doValueMapping?: boolean, wideLayoutIndexPath: number[] = [])
|
||||
{
|
||||
this.field = field;
|
||||
this.tableStructure = tableStructure;
|
||||
this.valueType = valueType;
|
||||
this.columnIndex = columnIndex;
|
||||
this.headerName = headerName;
|
||||
this.defaultValue = defaultValue;
|
||||
this.doValueMapping = doValueMapping;
|
||||
this.wideLayoutIndexPath = wideLayoutIndexPath;
|
||||
this.key = new Date().getTime().toString();
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public static clone(source: BulkLoadField): BulkLoadField
|
||||
{
|
||||
return (new BulkLoadField(source.field, source.tableStructure, source.valueType, source.columnIndex, source.headerName, source.defaultValue, source.doValueMapping, source.wideLayoutIndexPath));
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public getQualifiedName(): string
|
||||
{
|
||||
if (this.tableStructure.isMain)
|
||||
{
|
||||
return this.field.name;
|
||||
}
|
||||
|
||||
return this.tableStructure.associationPath + "." + this.field.name;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public getQualifiedNameWithWideSuffix(): string
|
||||
{
|
||||
let wideLayoutSuffix = "";
|
||||
if (this.wideLayoutIndexPath.length > 0)
|
||||
{
|
||||
wideLayoutSuffix = "." + this.wideLayoutIndexPath.map(i => i + 1).join(".");
|
||||
}
|
||||
|
||||
if (this.tableStructure.isMain)
|
||||
{
|
||||
return this.field.name + wideLayoutSuffix;
|
||||
}
|
||||
|
||||
return this.tableStructure.associationPath + "." + this.field.name + wideLayoutSuffix;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public getKey(): string
|
||||
{
|
||||
let wideLayoutSuffix = "";
|
||||
if (this.wideLayoutIndexPath.length > 0)
|
||||
{
|
||||
wideLayoutSuffix = "." + this.wideLayoutIndexPath.map(i => i + 1).join(".");
|
||||
}
|
||||
|
||||
if (this.tableStructure.isMain)
|
||||
{
|
||||
return this.field.name + wideLayoutSuffix + this.key;
|
||||
}
|
||||
|
||||
return this.tableStructure.associationPath + "." + this.field.name + wideLayoutSuffix + this.key;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public getQualifiedLabel(): string
|
||||
{
|
||||
let wideLayoutSuffix = "";
|
||||
if (this.wideLayoutIndexPath.length > 0)
|
||||
{
|
||||
wideLayoutSuffix = " (" + this.wideLayoutIndexPath.map(i => i + 1).join(", ") + ")";
|
||||
}
|
||||
|
||||
if (this.tableStructure.isMain)
|
||||
{
|
||||
return this.field.label + wideLayoutSuffix;
|
||||
}
|
||||
|
||||
return this.tableStructure.label + ": " + this.field.label + wideLayoutSuffix;
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public isMany(): boolean
|
||||
{
|
||||
return this.tableStructure && this.tableStructure.isMany;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** this is a type defined in qqq backend - a representation of a bulk-load
|
||||
** table - e.g., how it fits into qqq - and of note - how child / association
|
||||
** tables are nested too.
|
||||
***************************************************************************/
|
||||
export interface BulkLoadTableStructure
|
||||
{
|
||||
isMain: boolean;
|
||||
isMany: boolean;
|
||||
tableName: string;
|
||||
label: string;
|
||||
associationPath: string;
|
||||
fields: QFieldMetaData[];
|
||||
associations: BulkLoadTableStructure[];
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** this is the internal data structure that the UI works with - but notably,
|
||||
** is not how we send it to the backend or how backend saves profiles -- see
|
||||
** BulkLoadProfile for that.
|
||||
*******************************************************************************/
|
||||
export class BulkLoadMapping
|
||||
{
|
||||
fields: { [qualifiedName: string]: BulkLoadField } = {};
|
||||
fieldsByTablePrefix: { [prefix: string]: { [qualifiedFieldName: string]: BulkLoadField } } = {};
|
||||
tablesByPath: { [path: string]: BulkLoadTableStructure } = {};
|
||||
|
||||
requiredFields: BulkLoadField[] = [];
|
||||
additionalFields: BulkLoadField[] = [];
|
||||
unusedFields: BulkLoadField[] = [];
|
||||
|
||||
valueMappings: { [fieldName: string]: { [fileValue: string]: any } } = {};
|
||||
|
||||
hasHeaderRow: boolean;
|
||||
layout: string;
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
constructor(tableStructure: BulkLoadTableStructure)
|
||||
{
|
||||
if (tableStructure)
|
||||
{
|
||||
this.processTableStructure(tableStructure);
|
||||
|
||||
if (!tableStructure.associations)
|
||||
{
|
||||
this.layout = "FLAT";
|
||||
}
|
||||
}
|
||||
|
||||
this.hasHeaderRow = true;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
private processTableStructure(tableStructure: BulkLoadTableStructure)
|
||||
{
|
||||
const prefix = tableStructure.isMain ? "" : tableStructure.associationPath;
|
||||
this.fieldsByTablePrefix[prefix] = {};
|
||||
this.tablesByPath[prefix] = tableStructure;
|
||||
|
||||
for (let field of tableStructure.fields)
|
||||
{
|
||||
// todo delete this - backend should only give it to us if editable: if (field.isEditable)
|
||||
{
|
||||
const bulkLoadField = new BulkLoadField(field, tableStructure);
|
||||
const qualifiedName = bulkLoadField.getQualifiedName();
|
||||
this.fields[qualifiedName] = bulkLoadField;
|
||||
this.fieldsByTablePrefix[prefix][qualifiedName] = bulkLoadField;
|
||||
|
||||
if (tableStructure.isMain && field.isRequired)
|
||||
{
|
||||
this.requiredFields.push(bulkLoadField);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.unusedFields.push(bulkLoadField);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (let associatedTableStructure of tableStructure.associations ?? [])
|
||||
{
|
||||
this.processTableStructure(associatedTableStructure);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** take a saved bulk load profile - and convert it into a working bulkLoadMapping
|
||||
** for the frontend to use!
|
||||
***************************************************************************/
|
||||
public static fromSavedProfileRecord(tableStructure: BulkLoadTableStructure, profileRecord: QRecord): BulkLoadMapping
|
||||
{
|
||||
const bulkLoadProfile = JSON.parse(profileRecord.values.get("mappingJson")) as BulkLoadProfile;
|
||||
return BulkLoadMapping.fromBulkLoadProfile(tableStructure, bulkLoadProfile);
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** take a saved bulk load profile - and convert it into a working bulkLoadMapping
|
||||
** for the frontend to use!
|
||||
***************************************************************************/
|
||||
public static fromBulkLoadProfile(tableStructure: BulkLoadTableStructure, bulkLoadProfile: BulkLoadProfile): BulkLoadMapping
|
||||
{
|
||||
const bulkLoadMapping = new BulkLoadMapping(tableStructure);
|
||||
|
||||
if (bulkLoadProfile.version == "v1")
|
||||
{
|
||||
bulkLoadMapping.hasHeaderRow = bulkLoadProfile.hasHeaderRow;
|
||||
bulkLoadMapping.layout = bulkLoadProfile.layout;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// function to get a bulkLoadMapping field by its (full) name - whether that's in the required fields list, //
|
||||
// or it's an additional field, in which case, we'll go through the addField method to move what list it's in //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
function getBulkLoadFieldMovingFromUnusedToActiveIfNeeded(bulkLoadMapping: BulkLoadMapping, name: string): BulkLoadField
|
||||
{
|
||||
let wideIndex: number = null;
|
||||
if (name.match(/,\d+$/))
|
||||
{
|
||||
wideIndex = Number(name.match(/\d+$/));
|
||||
name = name.replace(/,\d+$/, "");
|
||||
}
|
||||
|
||||
for (let field of bulkLoadMapping.requiredFields)
|
||||
{
|
||||
if (field.getQualifiedName() == name)
|
||||
{
|
||||
return (field);
|
||||
}
|
||||
}
|
||||
|
||||
for (let field of bulkLoadMapping.unusedFields)
|
||||
{
|
||||
if (field.getQualifiedName() == name)
|
||||
{
|
||||
const addedField = bulkLoadMapping.addField(field, wideIndex);
|
||||
return (addedField);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// loop over fields in the profile - adding them to the mapping //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
for (let bulkLoadProfileField of ((bulkLoadProfile.fieldList ?? []) as BulkLoadProfileField[]))
|
||||
{
|
||||
const bulkLoadField = getBulkLoadFieldMovingFromUnusedToActiveIfNeeded(bulkLoadMapping, bulkLoadProfileField.fieldName);
|
||||
if (!bulkLoadField)
|
||||
{
|
||||
console.log(`Couldn't find bulk-load-field by name from profile record [${bulkLoadProfileField.fieldName}]`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((bulkLoadProfileField.columnIndex != null && bulkLoadProfileField.columnIndex != undefined) || (bulkLoadProfileField.headerName != null && bulkLoadProfileField.headerName != undefined))
|
||||
{
|
||||
bulkLoadField.valueType = "column";
|
||||
bulkLoadField.doValueMapping = bulkLoadProfileField.doValueMapping;
|
||||
bulkLoadField.headerName = bulkLoadProfileField.headerName;
|
||||
bulkLoadField.columnIndex = bulkLoadProfileField.columnIndex;
|
||||
|
||||
if (bulkLoadProfileField.valueMappings)
|
||||
{
|
||||
bulkLoadMapping.valueMappings[bulkLoadProfileField.fieldName] = {};
|
||||
for (let fileValue in bulkLoadProfileField.valueMappings)
|
||||
{
|
||||
////////////////////////////////////////////////////
|
||||
// frontend wants string values here, so, string. //
|
||||
////////////////////////////////////////////////////
|
||||
bulkLoadMapping.valueMappings[bulkLoadProfileField.fieldName][String(fileValue)] = bulkLoadProfileField.valueMappings[fileValue];
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
bulkLoadField.valueType = "defaultValue";
|
||||
bulkLoadField.defaultValue = bulkLoadProfileField.defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
return (bulkLoadMapping);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw ("Unexpected version for bulk load profile: " + bulkLoadProfile.version);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** take a working bulkLoadMapping from the frontend, and convert it to a
|
||||
** BulkLoadProfile for the backend / for us to save.
|
||||
***************************************************************************/
|
||||
public toProfile(): { haveErrors: boolean, profile: BulkLoadProfile }
|
||||
{
|
||||
let haveErrors = false;
|
||||
const profile = new BulkLoadProfile();
|
||||
|
||||
profile.version = "v1";
|
||||
profile.hasHeaderRow = this.hasHeaderRow;
|
||||
profile.layout = this.layout;
|
||||
|
||||
for (let bulkLoadField of [...this.requiredFields, ...this.additionalFields])
|
||||
{
|
||||
let fullFieldName = (bulkLoadField.tableStructure.isMain ? "" : bulkLoadField.tableStructure.associationPath + ".") + bulkLoadField.field.name;
|
||||
if (bulkLoadField.wideLayoutIndexPath != null && bulkLoadField.wideLayoutIndexPath != undefined && bulkLoadField.wideLayoutIndexPath.length)
|
||||
{
|
||||
fullFieldName += "," + bulkLoadField.wideLayoutIndexPath.join(".");
|
||||
}
|
||||
|
||||
bulkLoadField.error = null;
|
||||
if (bulkLoadField.valueType == "column")
|
||||
{
|
||||
if (bulkLoadField.columnIndex == undefined || bulkLoadField.columnIndex == null)
|
||||
{
|
||||
haveErrors = true;
|
||||
bulkLoadField.error = "You must select a column.";
|
||||
}
|
||||
else
|
||||
{
|
||||
const field: BulkLoadProfileField = {fieldName: fullFieldName, columnIndex: bulkLoadField.columnIndex, headerName: bulkLoadField.headerName, doValueMapping: bulkLoadField.doValueMapping};
|
||||
|
||||
if (this.valueMappings[fullFieldName])
|
||||
{
|
||||
field.valueMappings = this.valueMappings[fullFieldName];
|
||||
}
|
||||
|
||||
profile.fieldList.push(field);
|
||||
}
|
||||
}
|
||||
else if (bulkLoadField.valueType == "defaultValue")
|
||||
{
|
||||
if (bulkLoadField.defaultValue == undefined || bulkLoadField.defaultValue == null || bulkLoadField.defaultValue == "")
|
||||
{
|
||||
haveErrors = true;
|
||||
bulkLoadField.error = "A value is required.";
|
||||
}
|
||||
else
|
||||
{
|
||||
profile.fieldList.push({fieldName: fullFieldName, defaultValue: bulkLoadField.defaultValue});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {haveErrors, profile};
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public addField(bulkLoadField: BulkLoadField, specifiedWideIndex?: number): BulkLoadField
|
||||
{
|
||||
if (bulkLoadField.isMany() && this.layout == "WIDE")
|
||||
{
|
||||
let index: number;
|
||||
if (specifiedWideIndex != null && specifiedWideIndex != undefined)
|
||||
{
|
||||
index = specifiedWideIndex;
|
||||
}
|
||||
else
|
||||
{
|
||||
index = 0;
|
||||
///////////////////////////////////////////////////////////
|
||||
// count how many copies of this field there are already //
|
||||
///////////////////////////////////////////////////////////
|
||||
for (let existingField of [...this.requiredFields, ...this.additionalFields])
|
||||
{
|
||||
if (existingField.getQualifiedName() == bulkLoadField.getQualifiedName())
|
||||
{
|
||||
index++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const cloneField = BulkLoadField.clone(bulkLoadField);
|
||||
cloneField.wideLayoutIndexPath = [index];
|
||||
this.additionalFields.push(cloneField);
|
||||
return (cloneField);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.additionalFields.push(bulkLoadField);
|
||||
return (bulkLoadField);
|
||||
}
|
||||
}
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public removeField(toRemove: BulkLoadField): void
|
||||
{
|
||||
const newAdditionalFields: BulkLoadField[] = [];
|
||||
for (let bulkLoadField of this.additionalFields)
|
||||
{
|
||||
if (bulkLoadField.getQualifiedName() != toRemove.getQualifiedName())
|
||||
{
|
||||
newAdditionalFields.push(bulkLoadField);
|
||||
}
|
||||
}
|
||||
|
||||
this.additionalFields = newAdditionalFields;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** meta-data about the file that the user uploaded
|
||||
***************************************************************************/
|
||||
export class FileDescription
|
||||
{
|
||||
headerValues: string[];
|
||||
headerLetters: string[];
|
||||
bodyValuesPreview: string[][];
|
||||
|
||||
// todo - just get this from the profile always - it's not part of the file per-se
|
||||
hasHeaderRow: boolean = true;
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
constructor(headerValues: string[], headerLetters: string[], bodyValuesPreview: string[][])
|
||||
{
|
||||
this.headerValues = headerValues;
|
||||
this.headerLetters = headerLetters;
|
||||
this.bodyValuesPreview = bodyValuesPreview;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public setHasHeaderRow(hasHeaderRow: boolean)
|
||||
{
|
||||
this.hasHeaderRow = hasHeaderRow;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public getColumnNames(): string[]
|
||||
{
|
||||
if (this.hasHeaderRow)
|
||||
{
|
||||
return this.headerValues;
|
||||
}
|
||||
else
|
||||
{
|
||||
return this.headerLetters.map(l => `Column ${l}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public getPreviewValues(columnIndex: number): string[]
|
||||
{
|
||||
if (columnIndex == undefined)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
if (this.hasHeaderRow)
|
||||
{
|
||||
return (this.bodyValuesPreview[columnIndex]);
|
||||
}
|
||||
else
|
||||
{
|
||||
return ([this.headerValues[columnIndex], ...this.bodyValuesPreview[columnIndex]]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** this (BulkLoadProfile & ...Field) is the model of what we save, and is
|
||||
** also what we submit to the backend during the process.
|
||||
***************************************************************************/
|
||||
export class BulkLoadProfile
|
||||
{
|
||||
version: string;
|
||||
fieldList: BulkLoadProfileField[] = [];
|
||||
hasHeaderRow: boolean;
|
||||
layout: string;
|
||||
}
|
||||
|
||||
type BulkLoadProfileField =
|
||||
{
|
||||
fieldName: string,
|
||||
columnIndex?: number,
|
||||
headerName?: string,
|
||||
defaultValue?: any,
|
||||
doValueMapping?: boolean,
|
||||
valueMappings?: { [fileValue: string]: any }
|
||||
};
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** In the bulk load forms, we have some forward-ref callback functions, and
|
||||
** they like to capture/retain a reference when those functions get defined,
|
||||
** so we had some trouble updating objects in those functions.
|
||||
**
|
||||
** We "solved" this by creating instances of this class, which get captured,
|
||||
** so then we can replace the wrapped object, and have a better time...
|
||||
***************************************************************************/
|
||||
export class Wrapper<T>
|
||||
{
|
||||
t: T;
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
constructor(t: T)
|
||||
{
|
||||
this.t = t;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public get(): T
|
||||
{
|
||||
return this.t;
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
public set(t: T)
|
||||
{
|
||||
this.t = t;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user