From c3b5e951331df38e2eec4c5ddc2b56ccae53b7dd Mon Sep 17 00:00:00 2001 From: Offroaders123 <65947371+Offroaders123@users.noreply.github.com> Date: Tue, 16 May 2023 23:36:22 -0700 Subject: [PATCH] Compression Upgrade Did more investigating into multiple ways of using the Compression Streams API, and I realized that this is a really nice setup to have, possibly for a standalone module outside of NBTify. This provides a more cross-platform zlib API which would be similar to using `node:zlib`, but available both in the browser, and Node! It's build off of the Compression Streams API too, so it's only an API wrapper, and possibly a polyfill where necessary. I'm going to try settings things up first here, then move them to a dedicated project which both handles the compression polyfills, and provides these more functional-based APIs, which both provide more performance than my previous streams-specific `Blob` handling that I had before, and are simpler to use directly with `ArrayBuffer` and the various `TypedArray` objects, rather than having to go back and forth between `Blob` containers, which does appear to be a little less performant with the simple test that I set up. Oh my, that last sentence was horrible, I'm sorry. I guess I'm more of a dev than I am a writer. https://github.com/Offroaders123/Dovetail/issues/1 https://github.com/WICG/compression/issues/8 https://wicg.github.io/compression/#example-deflate-compress https://jakearchibald.com/2017/async-iterators-and-generators/#making-streams-iterate --- src/compression.ts | 94 ++++++++++++++++++++++++++++++++++------------ src/index.ts | 3 +- src/read.ts | 6 +-- src/write.ts | 6 +-- test/index.ts | 27 ++----------- test/types-demo.ts | 24 ++++++++++++ 6 files changed, 104 insertions(+), 56 deletions(-) create mode 100644 test/types-demo.ts diff --git a/src/compression.ts b/src/compression.ts index 053b541..a5ca09e 100644 --- a/src/compression.ts +++ b/src/compression.ts @@ -1,25 +1,71 @@ -export type CompressionFormat = "gzip" | "deflate" | "deflate-raw"; - -export interface CompressionOptions { - format: CompressionFormat; -} - -/** - * Compresses a Uint8Array using a specific compression format. -*/ -export async function compress(data: Uint8Array | ArrayBufferLike, { format }: CompressionOptions): Promise { - const { body } = new Response(data instanceof Uint8Array ? data : new Uint8Array(data)); - const readable = body!.pipeThrough(new CompressionStream(format)); - const buffer = await new Response(readable).arrayBuffer(); - return new Uint8Array(buffer); -} - -/** - * Decompresses a Uint8Array using a specific decompression format. -*/ -export async function decompress(data: Uint8Array | ArrayBufferLike, { format }: CompressionOptions): Promise { - const { body } = new Response(data instanceof Uint8Array ? data : new Uint8Array(data)); - const readable = body!.pipeThrough(new DecompressionStream(format)); - const buffer = await new Response(readable).arrayBuffer(); - return new Uint8Array(buffer); +export async function gzip(data: BufferSource): Promise { + return compress(data,"gzip"); +} + +export async function gunzip(data: BufferSource): Promise { + return decompress(data,"gzip"); +} + +export async function deflate(data: BufferSource): Promise { + return compress(data,"deflate"); +} + +export async function inflate(data: BufferSource): Promise { + return decompress(data,"deflate"); +} + +export async function deflateRaw(data: BufferSource): Promise { + return compress(data,"deflate-raw"); +} + +export async function inflateRaw(data: BufferSource): Promise { + return decompress(data,"deflate-raw"); +} + +async function compress(data: BufferSource, format: CompressionFormat): Promise { + const compressionStream = new CompressionStream(format); + return pipeThroughCompressionStream(data,compressionStream); +} + +async function decompress(data: BufferSource, format: CompressionFormat): Promise { + const decompressionStream = new DecompressionStream(format); + return pipeThroughCompressionStream(data,decompressionStream); +} + +async function pipeThroughCompressionStream(data: BufferSource, compressionStream: CompressionStream | DecompressionStream): Promise { + const writer = compressionStream.writable.getWriter(); + + writer.write(data); + writer.close(); + + const chunks: Uint8Array[] = []; + let byteLength = 0; + + for await (const chunk of readableStreamToAsyncGenerator(compressionStream.readable)){ + chunks.push(chunk); + byteLength += chunk.byteLength; + } + + const result = new Uint8Array(byteLength); + let byteOffset = 0; + + for (const chunk of chunks){ + result.set(chunk,byteOffset); + byteOffset += chunk.byteLength; + } + + return result; +} + +async function* readableStreamToAsyncGenerator(stream: ReadableStream): AsyncGenerator { + const reader = stream.getReader(); + try { + while (true){ + const { done, value } = await reader.read(); + if (done) return; + yield value; + } + } finally { + reader.releaseLock(); + } } \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index ad2d6a6..357bf10 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,5 +5,4 @@ export * from "./stringify.js"; export * from "./definition.js"; export * from "./data.js"; export * from "./tag.js"; -export * from "./primitive.js"; -export * from "./compression.js"; \ No newline at end of file +export * from "./primitive.js"; \ No newline at end of file diff --git a/src/read.ts b/src/read.ts index c27b487..c05babb 100644 --- a/src/read.ts +++ b/src/read.ts @@ -1,7 +1,7 @@ import { Name, Endian, Compression, BedrockLevel, NBTData } from "./data.js"; import { Int8, Int16, Int32, Float32 } from "./primitive.js"; import { TAG } from "./tag.js"; -import { decompress } from "./compression.js"; +import { gunzip, inflate } from "./compression.js"; import type { Tag, ListTag, CompoundTag } from "./tag.js"; @@ -79,11 +79,11 @@ export async function read(data: Uint8Array | ArrayBuffe } if (compression === "gzip"){ - data = await decompress(data,{ format: "gzip" }); + data = await gunzip(data); } if (compression === "deflate"){ - data = await decompress(data,{ format: "deflate" }); + data = await inflate(data); } if (bedrockLevel === undefined){ diff --git a/src/write.ts b/src/write.ts index a84b46d..75b7574 100644 --- a/src/write.ts +++ b/src/write.ts @@ -1,7 +1,7 @@ import { Name, Endian, Compression, BedrockLevel, NBTData } from "./data.js"; import { TAG, getTagType } from "./tag.js"; import { Int32 } from "./primitive.js"; -import { compress } from "./compression.js"; +import { gzip, deflate } from "./compression.js"; import type { RootTag, Tag, ByteTag, BooleanTag, ShortTag, IntTag, LongTag, FloatTag, DoubleTag, ByteArrayTag, StringTag, ListTag, CompoundTag, IntArrayTag, LongArrayTag } from "./tag.js"; @@ -57,11 +57,11 @@ export async function write(data: RootTag | NBTData, { name, endian, compression } if (compression === "gzip"){ - result = await compress(result,{ format: "gzip" }); + result = await gzip(result); } if (compression === "deflate"){ - result = await compress(result,{ format: "deflate" }); + result = await deflate(result); } return result; diff --git a/test/index.ts b/test/index.ts index 021cfa0..77f61cc 100644 --- a/test/index.ts +++ b/test/index.ts @@ -12,30 +12,9 @@ console.log(data,"\n"); const result = await NBT.read(data,{ strict: false }); console.log(result,"\n"); +console.log(result.data.SelectedItem.id,"\n"); + const recompile = await NBT.write(result).then(Buffer.from); console.log(recompile,"\n"); -console.log(Buffer.compare(data,recompile)); - -interface TempTest { - noice: NBT.BooleanTag; -} - -const { data: tempTest } = new NBT.NBTData({ - noice: true -}); - -tempTest.noice -// @ts-expect-error -tempTest.notAProperty - -const demo = new NBT.NBTData({ nice: true, smartTypes: 10 }); - -demo.data.smartTypes; - -interface MyData { Version: boolean; } -declare const heya: NBT.NBTData; -const noice = new NBT.NBTData(heya); - -const noice2 = await NBT.read(new Uint8Array(),heya); -const noice3 = new NBT.NBTReader().read(new Uint8Array(),heya); \ No newline at end of file +console.log(Buffer.compare(data,recompile)); \ No newline at end of file diff --git a/test/types-demo.ts b/test/types-demo.ts new file mode 100644 index 0000000..ca7f81c --- /dev/null +++ b/test/types-demo.ts @@ -0,0 +1,24 @@ +import * as NBT from "../src/index.js"; + +interface TempTest { + noice: NBT.BooleanTag; +} + +const { data: tempTest } = new NBT.NBTData({ + noice: true +}); + +tempTest.noice +// @ts-expect-error +tempTest.notAProperty + +const demo = new NBT.NBTData({ nice: true, smartTypes: 10 }); + +demo.data.smartTypes; + +interface MyData { Version: boolean; } +declare const heya: NBT.NBTData; +const noice = new NBT.NBTData(heya); + +const noice2 = await NBT.read(new Uint8Array(),heya); +const noice3 = new NBT.NBTReader().read(new Uint8Array(),heya); \ No newline at end of file