Export toJson

hyparam · Jan 5, 2024 · 954671f · 954671f
1 parent 02c2505
commit 954671f
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 30 deletions.
diff --git a/index.html b/index.html
@@ -31,7 +31,7 @@
   <body>
     <div id="dropzone">Drop .parquet file here</div>
     <script type="module">
-      import { parquetMetadata, toJson } from './src/metadata.js'
+      import { parquetMetadata, toJson } from './src/hyparquet.js'
 
       const dropZone = document.getElementById('dropzone')
 

diff --git a/package.json b/package.json
@@ -4,7 +4,9 @@
   "description": "parquet file parser for javascript",
   "keywords": [
     "parquet",
-    "parser"
+    "parser",
+    "snappy",
+    "thrift"
   ],
   "license": "MIT",
   "repository": {

diff --git a/src/hyparquet.d.ts b/src/hyparquet.d.ts
@@ -25,3 +25,13 @@ export function parquetMetadata(arrayBuffer: ArrayBuffer): any
  * @returns {boolean} true if successful
  */
 export function snappyUncompress(inputArray: Uint8Array, outputArray: Uint8Array): boolean
+
+/**
+ * Replace bigints with numbers.
+ * When parsing parquet files, bigints are used to represent 64-bit integers.
+ * However, JSON does not support bigints, so it's helpful to convert to numbers.
+ *
+ * @param {any} obj object to convert
+ * @returns {unknown} converted object
+ */
+export function toJson(obj: any): unknown
diff --git a/src/hyparquet.js b/src/hyparquet.js
@@ -4,6 +4,9 @@ export { parquetMetadata }
 import { snappyUncompress } from './snappy.js'
 export { snappyUncompress }
 
+import { toJson } from './toJson.js'
+export { toJson }
+
 /**
  * Read parquet data rows from a file
  *

diff --git a/src/metadata.js b/src/metadata.js
@@ -97,28 +97,3 @@ export function parquetMetadata(arrayBuffer) {
     created_by,
   }
 }
-
-/**
- * Replace bigints with numbers.
- * When parsing parquet files, bigints are used to represent 64-bit integers.
- * However, JSON does not support bigints, so it's helpful to convert to numbers.
- *
- * @param {any} obj object to convert
- * @returns {unknown} converted object
- */
-export function toJson(obj) {
-  if (typeof obj === 'bigint') {
-    return Number(obj)
-  } else if (Array.isArray(obj)) {
-    return obj.map(toJson)
-  } else if (obj instanceof Object) {
-    /** @type {Record<string, unknown>} */
-    const newObj = {}
-    for (const key of Object.keys(obj)) {
-      newObj[key] = toJson(obj[key])
-    }
-    return newObj
-  } else {
-    return obj
-  }
-}
diff --git a/src/toJson.js b/src/toJson.js
@@ -0,0 +1,24 @@
+/**
+ * Replace bigints with numbers.
+ * When parsing parquet files, bigints are used to represent 64-bit integers.
+ * However, JSON does not support bigints, so it's helpful to convert to numbers.
+ *
+ * @param {any} obj object to convert
+ * @returns {unknown} converted object
+ */
+export function toJson(obj) {
+  if (typeof obj === 'bigint') {
+    return Number(obj)
+  } else if (Array.isArray(obj)) {
+    return obj.map(toJson)
+  } else if (obj instanceof Object) {
+    /** @type {Record<string, unknown>} */
+    const newObj = {}
+    for (const key of Object.keys(obj)) {
+      newObj[key] = toJson(obj[key])
+    }
+    return newObj
+  } else {
+    return obj
+  }
+}
diff --git a/src/types.d.ts b/src/types.d.ts
@@ -33,7 +33,7 @@ export enum ParquetType {
   BOOLEAN = 0,
   INT32 = 1,
   INT64 = 2,
-  INT96 = 3,
+  INT96 = 3, // deprecated
   FLOAT = 4,
   DOUBLE = 5,
   BYTE_ARRAY = 6,
@@ -93,7 +93,7 @@ export enum Encoding {
   PLAIN = 0,
   PLAIN_DICTIONARY = 2,
   RLE = 3,
-  BIT_PACKED = 4,
+  BIT_PACKED = 4, // deprecated
   DELTA_BINARY_PACKED = 5,
   DELTA_LENGTH_BYTE_ARRAY = 6,
   DELTA_BYTE_ARRAY = 7,

diff --git a/test/metadata.test.js b/test/metadata.test.js
@@ -1,6 +1,7 @@
 import { promises as fs } from 'fs'
 import { describe, expect, it } from 'vitest'
-import { parquetMetadata, toJson } from '../src/metadata.js'
+import { parquetMetadata } from '../src/metadata.js'
+import { toJson } from '../src/toJson.js'
 
 /**
  * Helper function to read .parquet file into ArrayBuffer