WIP: Use queryBinder for exporting aspects

packages/transformer/package.json

@@ -25,7 +25,7 @@
     "lint:no-tests": "eslint -f visualstudio --quiet \"./src/*.ts\" 1>&2",
     "lint:fix": "eslint --fix -f visualstudio --quiet \"./src/**/*.ts\" 1>&2",
     "lint:with-warnings": "eslint -f visualstudio \"./src/**/*.ts\" 1>&2",
-    "test": "mocha \"lib/cjs/test/**/*.test.js\" --timeout 8000 --require source-map-support/register",
+    "test": "mocha \"lib/cjs/test/**/*.test.js\" --timeout 800000 --require source-map-support/register",
     "no-internal-report": "no-internal-report src/**/*.ts"
   },
   "repository": {

packages/transformer/src/IModelExporter.ts

@@ -12,7 +12,7 @@ import {
   IModelHost, IModelJsNative, Model, RecipeDefinitionElement, Relationship,
 } from "@itwin/core-backend";
 import { AccessToken, assert, CompressedId64Set, DbResult, Id64, Id64String, IModelStatus, Logger, YieldManager } from "@itwin/core-bentley";
-import { ChangesetIndexOrId, CodeSpec, FontProps, IModel, IModelError } from "@itwin/core-common";
+import { ChangesetIndexOrId, CodeSpec, ElementAspectProps, FontProps, IModel, IModelError, QueryBinder, QueryRowFormat } from "@itwin/core-common";
 import { ECVersion, Schema, SchemaKey, SchemaLoader } from "@itwin/ecschema-metadata";
 import { TransformerLoggerCategory } from "./TransformerLoggerCategory";
 import type { InitFromExternalSourceAspectsArgs } from "./IModelTransformer";
@@ -117,6 +117,14 @@ export abstract class IModelExportHandler {
    */
   public onExportElementMultiAspects(_aspects: ElementMultiAspect[]): void { }
 
+  public onExportElementMultiAspect(_aspects: ElementMultiAspect): void { }
+
+  public onExportElementAspect(_sourceElementAspect: ElementAspect) {}
+
+  public preExportElementAspects(_elementId: Id64String): void {}
+
+  public shouldExportElementAspects(_elementId: Id64String): boolean { return true; }
+
   /** If `true` is returned, then the relationship will be exported.
    * @note This method can optionally be overridden to exclude an individual CodeSpec from the export. The base implementation always returns `true`.
    */
@@ -557,6 +565,10 @@ export class IModelExporter {
         statement.bindId("rootSubjectId", IModel.rootSubjectId);
       }
       while (DbResult.BE_SQLITE_ROW === statement.step()) {
+        if(statement.getValue(0).getId() === "0x2d"){
+          // eslint-disable-next-line no-console
+          console.log("elementId");
+        }
         await this.exportElement(statement.getValue(0).getId());
         await this._yieldManager.allowYield();
       }
@@ -625,6 +637,10 @@ export class IModelExporter {
    * @note This method is called from [[exportChanges]] and [[exportAll]], so it only needs to be called directly when exporting a subset of an iModel.
    */
   public async exportElement(elementId: Id64String): Promise<void> {
+    if(elementId === "0x2d"){
+      // eslint-disable-next-line no-console
+      console.log("elementId");
+    }
     if (!this.visitElements) {
       Logger.logTrace(loggerCategory, `visitElements=false, skipping exportElement(${elementId})`);
       return;
@@ -648,7 +664,10 @@ export class IModelExporter {
       await this.handler.preExportElement(element);
       this.handler.onExportElement(element, isUpdate);
       await this.trackProgress();
-      await this.exportElementAspects(elementId);
+      if (this.handler.shouldExportElementAspects(elementId)) {
+        this.handler.preExportElementAspects(elementId);
+        await this.exportElementAspects(elementId);
+      }
       return this.exportChildElements(elementId);
     }
   }
@@ -684,7 +703,7 @@ export class IModelExporter {
 
   /** Export ElementAspects from the specified element from the source iModel. */
   private async exportElementAspects(elementId: Id64String): Promise<void> {
-    const _uniqueAspects = await Promise.all(this.sourceDb.elements
+    await Promise.all(this.sourceDb.elements
       ._queryAspects(elementId, ElementUniqueAspect.classFullName, this._excludedElementAspectClassFullNames)
       .filter((a) => this.shouldExportElementAspect(a))
       .map(async (uniqueAspect: ElementUniqueAspect) => {
@@ -698,13 +717,74 @@ export class IModelExporter {
         }
       }));
 
-    const multiAspects = this.sourceDb.elements
-      ._queryAspects(elementId, ElementMultiAspect.classFullName, this._excludedElementAspectClassFullNames)
-      .filter((a) => this.shouldExportElementAspect(a));
+    // const multiAspects = this.sourceDb.elements
+    //   ._queryAspects(elementId, ElementMultiAspect.classFullName, this._excludedElementAspectClassFullNames)
+    //   .filter((a) => this.shouldExportElementAspect(a));
+    // if (multiAspects.length > 0) {
+    //   this.handler.onExportElementMultiAspects(multiAspects);
+    //   return this.trackProgress();
+    // }
+
+    const aspectClassNameIdMap = new Map<string, Id64String> ();
+    // const sql = `SELECT DISTINCT ECClassId as clId, (ec_classname(ECClassId)) as clname FROM ${ElementMultiAspect.classFullName} WHERE Element.Id=:elementId`;
+    this.sourceDb.withPreparedStatement(`SELECT DISTINCT ECClassId as clId, (ec_classname(ECClassId)) as clname FROM ${ElementMultiAspect.classFullName} WHERE Element.Id=:elementId`, (stmt) => {
+      stmt.bindId("elementId", elementId);
+      while (DbResult.BE_SQLITE_ROW === stmt.step()) {
+        aspectClassNameIdMap.set(stmt.getValue(1).getString(), stmt.getValue(0).getId());
+      }
+    });
 
-    if (multiAspects.length > 0) {
-      this.handler.onExportElementMultiAspects(multiAspects);
-      return this.trackProgress();
+    if(aspectClassNameIdMap.size === 0){
+      return;
+    }
+
+    // this._handler?.preExportElementAspects(elementId);
+    for (const [className, classId] of aspectClassNameIdMap){
+      if(this._excludedElementAspectClassFullNames.has(className)){
+        continue;
+      }
+      const sql = `SELECT * FROM ${className} WHERE Element.id=:elementId AND ECClassId=:classId`;
+      const queryReader = this.sourceDb.query(sql, new QueryBinder().bindId("elementId", elementId).bindId("classId", classId), { rowFormat: QueryRowFormat.UseJsPropertyNames });
+      // const queryReader22 = this.sourceDb.createQueryReader(sql, new QueryBinder().bindId("elementId", elementId).bindId("classId", classId), { rowFormat: QueryRowFormat.UseJsPropertyNames });
+      // const metadata = await queryReader22.getMetaData();
+      // metadata;
+      for await (const rawAspectProps of queryReader) {
+        const aspectProps: ElementAspectProps = { ...rawAspectProps, classFullName: className }; // add in property required by EntityProps
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (aspectProps as any).className = undefined; // clear property from SELECT * that we don't want in the final instance
+        const elementAspectEntity = this.sourceDb.constructEntity<ElementAspect>(aspectProps);
+        if (this.shouldExportElementAspect(elementAspectEntity)) {
+          this.handler.onExportElementMultiAspect(elementAspectEntity);
+          await this.trackProgress();
+        }
+      }
+    }
+  }
+
+  public async exportAspects(): Promise<void> {
+    const aspectClassNameIdMap = new Map<string, Id64String>();
+    const sql = `SELECT DISTINCT ECClassId as clId, (ec_classname(ECClassId)) as clname FROM ${ElementAspect.classFullName}`;
+    this.sourceDb.withPreparedStatement(sql, (statement) => {
+      while (DbResult.BE_SQLITE_ROW === statement.step()) {
+        aspectClassNameIdMap.set(statement.getValue(1).getString(), statement.getValue(0).getId());
+      }
+    });
+
+    for (const [className, classId] of aspectClassNameIdMap) {
+      if(this._excludedElementAspectClassFullNames.has(className))
+        continue;
+
+      const sql2 = `SELECT * FROM ${className} WHERE ECClassId = :classId ORDER BY Element.Id`;
+      const queryReader = this.sourceDb.query(sql2, new QueryBinder().bindId("classId", classId), { rowFormat: QueryRowFormat.UseJsPropertyNames });
+      for await (const rawAspectProps of queryReader) {
+        const aspectProps: ElementAspectProps = { ...rawAspectProps, classFullName: rawAspectProps.className.replace(".", ":") }; // add in property required by EntityProps
+        (aspectProps as any).className = undefined; // clear property from SELECT * that we don't want in the final instance
+        const aspectEntity = this.sourceDb.constructEntity<ElementAspect>(aspectProps);
+        if (this.handler.shouldExportElementAspect(aspectEntity)) {
+          this.handler.onExportElementAspect(aspectEntity);
+        }
+        await this.trackProgress();
+      }
     }
   }
 

packages/transformer/src/IModelImporter.ts

@@ -382,6 +382,16 @@ export class IModelImporter implements Required<IModelImportOptions> {
     return result as Id64String[];
   }
 
+  public importElementMultiAspect(
+    aspectProps: ElementAspectProps
+  ): Id64String {
+    if (aspectProps.id) {
+      this.onUpdateElementAspect(aspectProps);
+      return aspectProps.id;
+    }
+    return this.onInsertElementAspect(aspectProps);
+  }
+
   /** Insert the ElementAspect into the target iModel.
    * @note A subclass may override this method to customize insert behavior but should call `super.onInsertElementAspect`.
    */

packages/transformer/src/IModelTransformer.ts

@@ -148,6 +148,8 @@ export interface IModelTransformOptions {
    * @beta
    */
   optimizeGeometry?: OptimizeGeometryOptions;
+
+  detachAspectProcessing?: boolean;
 }
 
 /**
@@ -1122,6 +1124,11 @@ export class IModelTransformer extends IModelExportHandler {
     return targetRelationshipProps;
   }
 
+  // TODO: this will only be called when processing aspects for each element separately.
+  public override shouldExportElementAspects(_elementId: Id64String): boolean {
+    return !this._options.detachAspectProcessing;
+  }
+
   /** Override of [IModelExportHandler.onExportElementUniqueAspect]($transformer) that imports an ElementUniqueAspect into the target iModel when it is exported from the source iModel.
    * This override calls [[onTransformElementAspect]] and then [IModelImporter.importElementUniqueAspect]($transformer) to update the target iModel.
    */
@@ -1134,6 +1141,36 @@ export class IModelTransformer extends IModelExportHandler {
     this.resolvePendingReferences(sourceAspect);
   }
 
+  private preExportAspects(): void {
+    const targetElementAspects = this.targetDb.withPreparedStatement(`SELECT EcInstanceId FROM ${ElementAspect.classFullName} WHERE ECInstanceId NOT IN (SELECT ECInstanceId FROM ${ExternalSourceAspect.classFullName} WHERE Scope.id=:targetScopeElementId) AND ECInstanceId NOT IN (SELECT ECInstanceId FROM BisCore:TextAnnotationData)`, (stmt) => {
+    //   stmt.bindId("elementId", targetElementId);
+      stmt.bindId("targetScopeElementId", this.targetScopeElementId);
+      const aspects: Id64String[] = [];
+      while (DbResult.BE_SQLITE_ROW === stmt.step()) {
+        aspects.push(stmt.getValue(0).getId());
+      }
+      return aspects;
+    });
+
+    this.targetDb.elements.deleteAspect(targetElementAspects);
+  }
+
+  public override preExportElementAspects(elementId: Id64String): void {
+    const targetElementId = this.context.findTargetElementId(elementId);
+
+    const targetElementAspects = this.targetDb.withPreparedStatement(`SELECT EcInstanceId FROM ${ElementMultiAspect.classFullName} WHERE Element.Id = :elementId AND ECInstanceId NOT IN (SELECT ECInstanceId FROM ${ExternalSourceAspect.classFullName} WHERE Element.Id = :elementId AND Scope.id=:targetScopeElementId)`, (stmt) => {
+      stmt.bindId("elementId", targetElementId);
+      stmt.bindId("targetScopeElementId", this.targetScopeElementId);
+      const aspects: Id64String[] = [];
+      while (DbResult.BE_SQLITE_ROW === stmt.step()) {
+        aspects.push(stmt.getValue(0).getId());
+      }
+      return aspects;
+    });
+
+    this.targetDb.elements.deleteAspect(targetElementAspects);
+  }
+
   /** Override of [IModelExportHandler.onExportElementMultiAspects]($transformer) that imports ElementMultiAspects into the target iModel when they are exported from the source iModel.
    * This override calls [[onTransformElementAspect]] for each ElementMultiAspect and then [IModelImporter.importElementMultiAspects]($transformer) to update the target iModel.
    * @note ElementMultiAspects are handled as a group to make it easier to differentiate between insert, update, and delete.
@@ -1154,6 +1191,37 @@ export class IModelTransformer extends IModelExportHandler {
     }
   }
 
+  public override onExportElementMultiAspect(sourceAspect: ElementMultiAspect): void {
+    const targetElementId: Id64String = this.context.findTargetElementId(sourceAspect.element.id);
+    // Transform source ElementMultiAspects into target ElementAspectProps
+    const targetAspectProps = this.onTransformElementAspect(sourceAspect, targetElementId);
+    this.collectUnmappedReferences(sourceAspect);
+
+    const isExternalSourceAspectFromTransformer = sourceAspect instanceof ExternalSourceAspect && (targetAspectProps as ExternalSourceAspectProps).scope?.id === this.targetScopeElementId;
+    if (!this._options.includeSourceProvenance || !isExternalSourceAspectFromTransformer) {
+      const aspectId = this.importer.importElementMultiAspect(targetAspectProps);
+      this.context.remapElementAspect(sourceAspect.id, aspectId);
+      this.resolvePendingReferences(sourceAspect);
+    }
+  }
+
+  public override onExportElementAspect(sourceElementAspect: ElementAspect) {
+    const targetElementId: Id64String = this.context.findTargetElementId(sourceElementAspect.element.id);
+    // Transform source ElementMultiAspects into target ElementAspectProps
+    const targetAspectProps = this.onTransformElementAspect(sourceElementAspect, targetElementId);
+    this.collectUnmappedReferences(sourceElementAspect);
+
+    const isExternalSourceAspectFromTransformer = sourceElementAspect instanceof ExternalSourceAspect && (targetAspectProps as ExternalSourceAspectProps).scope?.id === this.targetScopeElementId;
+    if (!this._options.includeSourceProvenance || !isExternalSourceAspectFromTransformer) {
+      if (sourceElementAspect instanceof ElementUniqueAspect) {
+        this.context.remapElementAspect(sourceElementAspect.id, this.importer.importElementUniqueAspect(targetAspectProps));
+      } else {
+        this.context.remapElementAspect(sourceElementAspect.id, this.importer.importElementMultiAspect(targetAspectProps));
+      }
+      this.resolvePendingReferences(sourceElementAspect);
+    }
+  }
+
   /** Transform the specified sourceElementAspect into ElementAspectProps for the target iModel.
    * @param sourceElementAspect The ElementAspect from the source iModel to be transformed.
    * @param _targetElementId The ElementId of the target Element that will own the ElementAspects after transformation.
@@ -1315,6 +1383,7 @@ export class IModelTransformer extends IModelExportHandler {
  * @note [[processSchemas]] is not called automatically since the target iModel may want a different collection of schemas.
  */
   public async processAll(): Promise<void> {
+    this._options.detachAspectProcessing = true;
     Logger.logTrace(loggerCategory, "processAll()");
     this.logSettings();
     this.validateScopeProvenance();
@@ -1326,6 +1395,10 @@ export class IModelTransformer extends IModelExportHandler {
     await this.exporter.exportModelContents(IModel.repositoryModelId, Element.classFullName, true); // after the Subject hierarchy, process the other elements of the RepositoryModel
     await this.exporter.exportSubModels(IModel.repositoryModelId); // start below the RepositoryModel
     await this.exporter.exportRelationships(ElementRefersToElements.classFullName);
+    if(this._options.detachAspectProcessing) {
+      this.preExportAspects();
+      await this.exporter.exportAspects();
+    }
     await this.processDeferredElements(); // eslint-disable-line deprecation/deprecation
     if (this.shouldDetectDeletes()) {
       await this.detectElementDeletes();