diff --git a/.gitignore b/.gitignore index b92fe71..448eff1 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,6 @@ npm-debug.log* yarn-debug.log* yarn-error.log* - docusaurus-theme-github-codeblock-main scene-screenshot-gen/index.js diff --git a/demo-snippets/tutorials/searching/index.ts b/demo-snippets/tutorials/searching/index.ts index 7d37a73..9c8a453 100644 --- a/demo-snippets/tutorials/searching/index.ts +++ b/demo-snippets/tutorials/searching/index.ts @@ -2,6 +2,8 @@ import searchByPath from "!!./search_by_path.ts?raw"; import fluffySearch from "!!./fluffy_search.ts?raw"; import exactSearch from "!!./exact_search.ts?raw"; import exactSearchExcludingResult from "!!./exact_search_excluding_result.ts?raw"; +import semanticSearch from "!!./semantic_search.ts?raw"; +import semanticSearch001 from "!!./semantic_search_001.ts?raw"; import { demo } from "../../misc"; @@ -10,4 +12,6 @@ export const searching = { ...demo("searching", "fluffySearch", fluffySearch, {}, 'Fluffy search pattern which will search all properties for words starting with "Roof".'), ...demo("searching", "exactSearch", exactSearch, {}, 'Exact search only checking the property "ifcClass" and the exact value "ifcRoof".'), ...demo("searching", "exactSearchExcludingResult", exactSearchExcludingResult, {}, "Same as the Exact search pattern, but with exclude. This will return all objects except the ones found above."), + ...demo("searching", "semanticSearch", semanticSearch, {}, "Semantic search test."), + ...demo("searching", "semanticSearch001", semanticSearch001, {}, "Semantic search test."), }; diff --git a/demo-snippets/tutorials/searching/semantic_search.ts b/demo-snippets/tutorials/searching/semantic_search.ts new file mode 100644 index 0000000..9f0f8a3 --- /dev/null +++ b/demo-snippets/tutorials/searching/semantic_search.ts @@ -0,0 +1,255 @@ +// HiddenRangeStarted +import * as WebglApi from "@novorender/webgl-api"; +import * as MeasureApi from "@novorender/measure-api"; +import * as DataJsApi from "@novorender/data-js-api"; +import * as GlMatrix from "gl-matrix"; + +export interface IParams { + webglApi: typeof WebglApi; + measureApi: typeof MeasureApi; + dataJsApi: typeof DataJsApi; + glMatrix: typeof GlMatrix; + canvas: HTMLCanvasElement; + canvas2D: HTMLCanvasElement; + previewCanvas: HTMLCanvasElement; +} + +// HiddenRangeEnded +export async function main({ webglApi, dataJsApi, canvas }: IParams) { + try { + // load scene into data api, create webgl api, view and load scene and set cameraController. + const view = await initView(webglApi, dataJsApi, canvas); + + const scene = view.scene!; + + // run render loop and canvas resizeObserver + run(view, canvas); + + createSearchUi(canvas.parentElement!, scene); + } catch (e) { + console.warn(e); + } +} + +function isolateObjects(scene: WebglApi.Scene, ids: number[]): void { + // Set highlight 255 on all objects + // Highlight index 255 is reserved fully transparent + scene.objectHighlighter.objectHighlightIndices.fill(255); + + // Set highlight back to 0 for objects to be isolated + // Highlight 0 should be neutral as we haven't changed view.settings.objectHighlights + ids.forEach((id) => (scene.objectHighlighter.objectHighlightIndices[id] = 0)); + + scene.objectHighlighter.commit(); +} +// HiddenRangeStarted +async function initView(webglApi: typeof WebglApi, dataJsAPI: typeof DataJsApi, canvas: HTMLCanvasElement): Promise { + // Initialize the data API with the Novorender data server service + const dataApi = dataJsAPI.createAPI({ + serviceUrl: "https://data.novorender.com/api", + }); + + // Load scene metadata + const sceneData = await dataApi + // Condos scene ID, but can be changed to any public scene ID + .loadScene("3b5e65560dc4422da5c7c3f827b6a77c") + .then((res) => { + if ("error" in res) { + throw res; + } else { + return res; + } + }); + + // Destructure relevant properties into variables + const { url, db, settings, camera: cameraParams } = sceneData; + + // initialize the webgl api + const api = webglApi.createAPI(); + + // Load scene + const scene = await api.loadScene(url, db); + + // Create a view with the scene's saved settings + const view = await api.createView(settings, canvas); + + // Set resolution scale to 1 + view.applySettings({ quality: { resolution: { value: 1 } } }); + + // Create a camera controller with the saved parameters with turntable as fallback + const camera = cameraParams ?? ({ kind: "turntable" } as any); + view.camera.controller = api.createCameraController(camera, canvas); + + // Assign the scene to the view + view.scene = scene; + + return view; +} + +async function run(view: WebglApi.View, canvas: HTMLCanvasElement): Promise { + // Handle canvas resizes + new ResizeObserver((entries) => { + for (const entry of entries) { + canvas.width = entry.contentRect.width; + canvas.height = entry.contentRect.height; + view.applySettings({ + display: { width: canvas.width, height: canvas.height }, + background: { color: [0, 0, 0.1, 1] }, + }); + } + }).observe(canvas); + + // Create a bitmap context to display render output + const ctx = canvas.getContext("bitmaprenderer"); + + // render loop + while (true) { + // Render frame + const output = await view.render(); + { + // Finalize output image + const image = await output.getImage(); + if (image) { + // Display the given ImageBitmap in the canvas associated with this rendering context. + ctx?.transferFromImageBitmap(image); + // release bitmap data + image.close(); + } + } + (output as any).dispose(); + } +} + +let props: Array<{ prop: string[]; isActive: boolean }> = []; + +// Function to create chip +function createChip(label: string, scene: WebglApi.Scene, chipWrapper: HTMLDivElement, isActive: boolean) { + // Create chip container div + const chipContainer = document.createElement("div"); + + chipContainer.classList.add("search-test-chip-element"); + + if (isActive) { + chipContainer.classList.add("is-active"); + } + + // Create chip span + const chipSpan = document.createElement("span"); + chipSpan.textContent = label + " "; + + // Add click event listener to chip close button + chipContainer.onclick = async () => { + props.forEach((p) => { + const currentLabel = label.split(": "); + if (p.prop[0] === currentLabel[0] && p.prop[1] === currentLabel[1]) { + p.isActive = !p.isActive; + } + }); + + await createChipElementsThenSearchAndIsolate(scene, chipWrapper); + }; + + // Append chip span and chip close button to chip container + chipContainer.appendChild(chipSpan); + + // Return chip container + return chipContainer; +} + +function createSearchUi(container: HTMLElement, scene: WebglApi.Scene) { + // Create container div + const wrapper = document.createElement("div"); + const chipWrapper = document.createElement("div"); + + wrapper.classList.add("search-test-wrapper-element"); + + // Create index name field + const namespace = document.createElement("input"); + namespace.type = "text"; + namespace.placeholder = "Enter pinecone index name"; + namespace.value = "condos"; + namespace.style.width = "25%"; + // Create input field + const input = document.createElement("input"); + input.type = "text"; + input.placeholder = "Enter search term..."; + + // Create search button + const searchButton = document.createElement("button"); + searchButton.textContent = "Search"; + + let noResultsMsg: HTMLParagraphElement; + + searchButton.onclick = async () => { + if (noResultsMsg) { + wrapper.removeChild(noResultsMsg); + } + + searchButton.textContent = "Loading..."; + + const response = await fetch(`https://novorender-semantic-search-test-api.onrender.com/search?namespace=${namespace.value}&query=${input.value}`); + const { res } = await response.json(); + // Replace all single quotes with double quotes so we can json parse + const validJSONStr = res.replace(/'/g, '"'); + + props = JSON.parse(validJSONStr).map((e: any) => ({ prop: e, isActive: true })); + + await createChipElementsThenSearchAndIsolate(scene, chipWrapper); + + if (!props.length) { + noResultsMsg = document.createElement("p"); + noResultsMsg.textContent = "No results found for you query, try rephrasing your query."; + wrapper.appendChild(noResultsMsg); + } + + searchButton.textContent = "Search"; + input.value = ""; + }; + + // Append input field and search button to container + wrapper.appendChild(namespace); + wrapper.appendChild(input); + wrapper.appendChild(searchButton); + wrapper.appendChild(chipWrapper); + + // Append container and chip container to the document body + container.appendChild(wrapper); +} + +function createChipElements(scene: WebglApi.Scene, chipWrapper: HTMLDivElement) { + chipWrapper.replaceChildren( + ...props.map((chip) => { + return createChip(`${chip.prop[0]}: ${chip.prop[1]}`, scene, chipWrapper, chip.isActive); + }) + ); +} + +async function createChipElementsThenSearchAndIsolate(scene: WebglApi.Scene, chipWrapper: HTMLDivElement) { + // reset and re-apply the search + scene.objectHighlighter.objectHighlightIndices.fill(0); + scene.objectHighlighter.commit(); + createChipElements(scene, chipWrapper); + if (props.filter((p) => p.isActive).length) { + await searchAndIsolate(scene); + } +} + +async function searchAndIsolate(scene: WebglApi.Scene) { + const filteredProps = props.filter((p) => p.isActive); + + // Run the searches + // Exact search only checking the property "ifcClass" and the exact value "ifcRoof" + const iterator = scene.search({ + searchPattern: filteredProps.map((i: any) => ({ property: i.prop[0], value: i.prop[1], exact: true })), + }); + + // In this example we just want to isolate the objects so all we need is the object ID + const result: number[] = []; + for await (const object of iterator) { + result.push(object.id); + } + + // Then we isolate the objects found + isolateObjects(scene, result); +} +// HiddenRangeEnded diff --git a/demo-snippets/tutorials/searching/semantic_search_001.ts b/demo-snippets/tutorials/searching/semantic_search_001.ts new file mode 100644 index 0000000..f5d3a04 --- /dev/null +++ b/demo-snippets/tutorials/searching/semantic_search_001.ts @@ -0,0 +1,324 @@ +// HiddenRangeStarted +import * as WebglApi from "@novorender/webgl-api"; +import * as MeasureApi from "@novorender/measure-api"; +import * as DataJsApi from "@novorender/data-js-api"; +import * as GlMatrix from "gl-matrix"; + +export interface IParams { + webglApi: typeof WebglApi; + measureApi: typeof MeasureApi; + dataJsApi: typeof DataJsApi; + glMatrix: typeof GlMatrix; + canvas: HTMLCanvasElement; + canvas2D: HTMLCanvasElement; + previewCanvas: HTMLCanvasElement; +} + +// HiddenRangeEnded +export async function main({ webglApi, dataJsApi, canvas }: IParams) { + // Hardcoded values for demo purposes + const username = ""; + const password = ""; + const sceneId = ""; + + const DATA_API_SERVICE_URL = "https://data.novorender.com/api"; + + // For the demo we have simplified the login flow to always run the login call + const accessToken = await login(username, password, DATA_API_SERVICE_URL); + + // Initialize the data API with the Novorender data server service + // and a callback which returns the auth header with the access token + const dataApi = dataJsApi.createAPI({ + serviceUrl: DATA_API_SERVICE_URL, + authHeader: async () => ({ + header: "Authorization", + // We are using pre-generated demo token here for brevity. + // To get your own token, look at "https://docs.novorender.com/data-rest-api/#/operations/Login". + value: `Bearer ${accessToken}`, + }), + }); + + // From here on everything except the scene ID is the same as for loading public scenes + + try { + // Load scene metadata + const sceneData = await dataApi + // Condos scene ID, but this one requires authentication + .loadScene(sceneId) + .then((res) => { + if ("error" in res) { + throw res; + } else { + return res; + } + }); + + // Destructure relevant properties into variables + const { url, db, settings, camera: cameraParams, objectGroups } = sceneData; + + // initialize the webgl api + const api = webglApi.createAPI(); + + // Load scene + const scene = await api.loadScene(url, db); + + // The code above is all you need to load the scene, + // however there is more scene data loaded that you can apply + + // Create a view with the scene's saved settings + const view = await api.createView(settings, canvas); + + // Set resolution scale to 1 + view.applySettings({ quality: { resolution: { value: 1 } } }); + + // Create a camera controller with the saved parameters with turntable as fallback + // available controller types are static, orbit, flight and turntable + const camera = cameraParams ?? ({ kind: "turntable" } as any); + const controller = api.createCameraController(camera, canvas); + controller.autoZoomToScene = !cameraParams; + view.camera.controller = controller; + + // Assign the scene to the view + view.scene = scene; + + // Run render loop and the resizeObserver + run(view, canvas); + + // For groups that have large .ids lists we have to explicitly load the IDs + // when needed as to not bloat the .loadScene() response + const groupIdRequests: Promise[] = objectGroups.map(async (group) => { + if ((group.selected || group.hidden) && !group.ids) { + group.ids = await dataApi.getGroupIds(sceneId, group.id).catch(() => { + console.warn("failed to load ids for group - ", group.id); + return []; + }); + } + }); + + await Promise.all(groupIdRequests); + + objectGroups.filter((group) => group.hidden).forEach((group) => group.ids?.forEach((id) => (scene.objectHighlighter.objectHighlightIndices[id] = 255))); + + scene.objectHighlighter.commit(); + + createSearchUi(canvas.parentElement!, scene, sceneId, dataApi, objectGroups); + } catch (e) { + // Handle errors however you like + console.warn(e); + } +} + +async function login(username: string, password: string, DATA_API_SERVICE_URL: string): Promise { + // POST to the data server service's /user/login endpoint + const res: { token: string } = await fetch(DATA_API_SERVICE_URL + "/user/login", { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: `username=${username}&password=${password}`, + }) + .then((res) => res.json()) + .catch(() => { + // Handle however you like + return { token: "" }; + }); + + return res.token; +} + +async function run(view: WebglApi.View, canvas: HTMLCanvasElement): Promise { + // Handle canvas resizes + new ResizeObserver((entries) => { + for (const entry of entries) { + canvas.width = entry.contentRect.width; + canvas.height = entry.contentRect.height; + view.applySettings({ + display: { width: canvas.width, height: canvas.height }, + }); + } + }).observe(canvas); + + // Create a bitmap context to display render output + const ctx = canvas.getContext("bitmaprenderer"); + + // render loop + while (true) { + // Render frame + const output = await view.render(); + { + // Finalize output image + const image = await output.getImage(); + if (image) { + // Display the given ImageBitmap in the canvas associated with this rendering context. + ctx?.transferFromImageBitmap(image); + // release bitmap data + image.close(); + } + } + (output as any).dispose(); + } +} + +let props: Array<{ prop: string[]; isActive: boolean }> = []; + +// Function to create chip +function createChip(label: string, scene: WebglApi.Scene, chipWrapper: HTMLDivElement, sceneId: any, dataApi: any, objectGroups: any, isActive: boolean) { + // Create chip container div + const chipContainer = document.createElement("div"); + + chipContainer.classList.add("search-test-chip-element"); + + if (isActive) { + chipContainer.classList.add("is-active"); + } + + // Create chip span + const chipSpan = document.createElement("span"); + chipSpan.textContent = label + " "; + + chipContainer.onclick = async () => { + props.forEach((p) => { + const currentLabel = label.split(": "); + if (p.prop[0] === currentLabel[0] && p.prop[1] === currentLabel[1]) { + p.isActive = !p.isActive; + } + }); + + await createChipElementsThenSearchAndIsolate(scene, chipWrapper, sceneId, dataApi, objectGroups); + }; + + // Append chip span and chip close button to chip container + chipContainer.appendChild(chipSpan); + + // Return chip container + return chipContainer; +} + +function createSearchUi(container: HTMLElement, scene: WebglApi.Scene, sceneId: any, dataApi: any, objectGroups: any) { + // Create container div + const wrapper = document.createElement("div"); + const chipWrapper = document.createElement("div"); + chipWrapper.style.maxHeight = "70px"; + chipWrapper.style.overflow = "auto"; + + wrapper.classList.add("search-test-wrapper-element"); + + // Create index name field + const namespace = document.createElement("input"); + namespace.type = "text"; + namespace.placeholder = "Enter pinecone index name"; + namespace.value = "banenor"; + namespace.style.width = "25%"; + // Create input field + const input = document.createElement("input"); + input.type = "text"; + input.placeholder = "Enter search term..."; + + // Create search button + const searchButton = document.createElement("button"); + searchButton.textContent = "Search"; + + let noResultsMsg: HTMLParagraphElement; + + searchButton.onclick = async () => { + if (noResultsMsg) { + wrapper.removeChild(noResultsMsg); + } + + searchButton.textContent = "Loading..."; + + const response = await fetch(`https://novorender-semantic-search-test-api.onrender.com/search?namespace=${namespace.value}&query=${input.value}`); + const { res } = await response.json(); + // Replace all single quotes with double quotes so we can json parse + const validJSONStr = res.replace(/'/g, '"'); + + props = JSON.parse(validJSONStr).map((e: any) => ({ prop: e, isActive: true })); + + await createChipElementsThenSearchAndIsolate(scene, chipWrapper, sceneId, dataApi, objectGroups); + + if (!props.length) { + noResultsMsg = document.createElement("p"); + noResultsMsg.textContent = "No results found for you query, try rephrasing your query."; + wrapper.appendChild(noResultsMsg); + } + + searchButton.textContent = "Search"; + input.value = ""; + }; + + // Append input field and search button to container + wrapper.appendChild(namespace); + wrapper.appendChild(input); + wrapper.appendChild(searchButton); + wrapper.appendChild(chipWrapper); + + // Append container and chip container to the document body + container.appendChild(wrapper); +} + +function createChipElements(scene: WebglApi.Scene, chipWrapper: HTMLDivElement, sceneId: any, dataApi: any, objectGroups: any) { + chipWrapper.replaceChildren( + ...props.map((chip) => { + return createChip(`${chip.prop[0]}: ${chip.prop[1]}`, scene, chipWrapper, sceneId, dataApi, objectGroups, chip.isActive); + }) + ); +} + +async function createChipElementsThenSearchAndIsolate(scene: WebglApi.Scene, chipWrapper: HTMLDivElement, sceneId: any, dataApi: any, objectGroups: any) { + // For groups that have large .ids lists we have to explicitly load the IDs + // when needed as to not bloat the .loadScene() response + const groupIdRequests: Promise[] = objectGroups.map(async (group: any) => { + if ((group.selected || group.hidden) && !group.ids) { + group.ids = await dataApi.getGroupIds(sceneId, group.id).catch(() => { + console.warn("failed to load ids for group - ", group.id); + return []; + }); + } + }); + + await Promise.all(groupIdRequests); + + scene.objectHighlighter.objectHighlightIndices.fill(0); + objectGroups.filter((group: any) => group.hidden).forEach((group: any) => group.ids?.forEach((id: any) => (scene.objectHighlighter.objectHighlightIndices[id] = 255))); + + scene.objectHighlighter.commit(); + + // reset and then re-apply the search + + // scene.objectHighlighter.commit(); + createChipElements(scene, chipWrapper, sceneId, dataApi, objectGroups); + if (props.filter((p) => p.isActive).length) { + await searchAndIsolate(scene); + } +} + +async function searchAndIsolate(scene: WebglApi.Scene) { + const filteredProps = props.filter((p) => p.isActive); + + // Run the searches + // Exact search only checking the property "ifcClass" and the exact value "ifcRoof" + const iterator = scene.search({ + searchPattern: filteredProps.map((i: any) => ({ property: i.prop[0], value: i.prop[1], exact: true })), + }); + + // In this example we just want to isolate the objects so all we need is the object ID + const result: number[] = []; + for await (const object of iterator) { + result.push(object.id); + } + + // Then we isolate the objects found + isolateObjects(scene, result); +} + +function isolateObjects(scene: WebglApi.Scene, ids: number[]): void { + // Set highlight 255 on all objects + // Highlight index 255 is reserved fully transparent + scene.objectHighlighter.objectHighlightIndices.fill(255); + + // Set highlight back to 0 for objects to be isolated + // Highlight 0 should be neutral as we haven't changed view.settings.objectHighlights + ids.forEach((id) => (scene.objectHighlighter.objectHighlightIndices[id] = 0)); + + scene.objectHighlighter.commit(); +} diff --git a/package-lock.json b/package-lock.json index afe3302..0a84a21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,23 +31,23 @@ "react-transition-group": "4.4.5" }, "devDependencies": { - "@docusaurus/eslint-plugin": "^2.3.1", + "@docusaurus/eslint-plugin": "2.3.1", "@docusaurus/module-type-aliases": "2.3.1", "@tsconfig/docusaurus": "1.0.6", "@tsconfig/node16": "1.0.3", "@types/puppeteer": "5.4.6", - "@typescript-eslint/eslint-plugin": "^5.54.0", - "@typescript-eslint/parser": "^5.54.0", + "@typescript-eslint/eslint-plugin": "5.54.0", + "@typescript-eslint/parser": "5.54.0", "ajv-cli": "5.0.0", "buffer": "6.0.3", "copy-webpack-plugin": "11.0.0", "css-minimizer-webpack-plugin": "4.1.0", "docusaurus-plugin-typedoc": "0.18.0", - "eslint": "^8.35.0", - "eslint-config-prettier": "^8.7.0", - "eslint-plugin-react": "^7.32.2", + "eslint": "8.35.0", + "eslint-config-prettier": "8.7.0", + "eslint-plugin-react": "7.32.2", "husky": "8.0.3", - "lint-staged": "^13.1.2", + "lint-staged": "13.1.2", "path-browserify": "1.0.1", "process": "0.11.10", "puppeteer": "13.5.2", diff --git a/package.json b/package.json index 6feb2ca..2eb705c 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "scripts": { "docusaurus": "docusaurus", "start": "docusaurus start", - "build": "npm run validate-snippets && docusaurus build", + "build": "npm run clear && npm run validate-snippets && docusaurus build", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", @@ -47,23 +47,23 @@ "react-transition-group": "4.4.5" }, "devDependencies": { - "@docusaurus/eslint-plugin": "^2.3.1", + "@docusaurus/eslint-plugin": "2.3.1", "@docusaurus/module-type-aliases": "2.3.1", "@tsconfig/docusaurus": "1.0.6", "@tsconfig/node16": "1.0.3", "@types/puppeteer": "5.4.6", - "@typescript-eslint/eslint-plugin": "^5.54.0", - "@typescript-eslint/parser": "^5.54.0", + "@typescript-eslint/eslint-plugin": "5.54.0", + "@typescript-eslint/parser": "5.54.0", "ajv-cli": "5.0.0", "buffer": "6.0.3", "copy-webpack-plugin": "11.0.0", "css-minimizer-webpack-plugin": "4.1.0", "docusaurus-plugin-typedoc": "0.18.0", - "eslint": "^8.35.0", - "eslint-config-prettier": "^8.7.0", - "eslint-plugin-react": "^7.32.2", + "eslint": "8.35.0", + "eslint-config-prettier": "8.7.0", + "eslint-plugin-react": "7.32.2", "husky": "8.0.3", - "lint-staged": "^13.1.2", + "lint-staged": "13.1.2", "path-browserify": "1.0.1", "process": "0.11.10", "puppeteer": "13.5.2", diff --git a/src/css/custom.css b/src/css/custom.css index 1826168..e8f6fff 100644 --- a/src/css/custom.css +++ b/src/css/custom.css @@ -210,3 +210,51 @@ opacity: 1; } } + +.search-test-wrapper-element { + position: absolute; + top: 0; + padding: 10px; + background: #3794ff; + left: 0; + right: 0; + margin: 0 auto; + width: 50%; + border-radius: 0 0 5px 5px; +} + +.search-test-wrapper-element input { + width: 60%; + border: 1px solid; + border-radius: 5px; + font-size: 12px; + margin-right: 6px; +} + +.search-test-wrapper-element input + button { + background: white; + color: gray; + border-radius: 5px; +} + +.search-test-chip-element { + font-size: 11px; + background: #3e34d3; + display: inline-block; + padding: 0 5px; + border-radius: 12px; + cursor: pointer; +} + +.search-test-chip-element:not(.is-active) { + filter: opacity(0.5); +} + +/* .search-test-chip-element button { + background: red; + border-radius: 10px; + font-size: 10px; + position: relative; + bottom: 1px; + left: 3px; +} */