From 981410f9df0b96ac347dc71d9c5b37a377997d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Wed, 22 May 2024 14:57:24 +0200 Subject: [PATCH 1/4] detect date fields in Apache Arrow tables --- package.json | 1 + src/table.js | 2 + test/arrow-test.js | 16 +++++ yarn.lock | 151 ++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 test/arrow-test.js diff --git a/package.json b/package.json index 9eb2a84..c52360d 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "@rollup/plugin-node-resolve": "13", "@rollup/plugin-replace": "3", "clean-css": "5", + "apache-arrow": "16", "d3": "7", "eslint": "8", "js-beautify": "1", diff --git a/src/table.js b/src/table.js index bd2df74..dd6096c 100644 --- a/src/table.js +++ b/src/table.js @@ -355,6 +355,8 @@ function alignof(base = {}, data, columns) { } function type(data, column) { + // duck-type Arrow table date fields + if (String(data?.schema?.fields?.find?.(d => d?.name === column)).match(//)) return "date"; for (const d of data) { if (d == null) continue; const value = d[column]; diff --git a/test/arrow-test.js b/test/arrow-test.js new file mode 100644 index 0000000..7667c64 --- /dev/null +++ b/test/arrow-test.js @@ -0,0 +1,16 @@ +import assert from "assert"; +import {table} from "@observablehq/inputs"; +import {tableFromJSON} from "apache-arrow"; +import {autoType, csv} from "d3"; +import it from "./jsdom.js"; + +it("Inputs.table() detects dates in Arrow tables", async () => { + const athletes = tableFromJSON(await csv("data/athletes.csv", autoType)); + const t = table(athletes); + const id = t.querySelector("td:nth-of-type(2)").innerHTML; + assert.strictEqual(id, "736,041,664"); + const name = t.querySelector("td:nth-of-type(3)").innerHTML; + assert.strictEqual(name, "A Jesus Garcia"); + const date = t.querySelector("td:nth-of-type(6)").innerHTML; + assert.strictEqual(date, "1969-10-17"); +}); diff --git a/yarn.lock b/yarn.lock index 81cedfa..2c1c4e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,14 @@ # yarn lockfile v1 +"@75lb/deep-merge@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@75lb/deep-merge/-/deep-merge-1.1.1.tgz#3b06155b90d34f5f8cc2107d796f1853ba02fd6d" + integrity sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw== + dependencies: + lodash.assignwith "^4.2.0" + typical "^7.1.1" + "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4": version "7.16.0" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.16.0.tgz#0dfc80309beec8411e65e706461c408b0bb9b431" @@ -274,6 +282,13 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.2.0.tgz#667bfc6186ae7c9e0b45a08960c551437176e1ca" integrity sha512-VkE3KLBmJwcCaVARtQpfuKcKv8gcBmUubrfHGF84dXuuW6jgsRYxPtzcIhPyK9WAPpRt2/xY6zkD9MnRaJzSyw== +"@swc/helpers@^0.5.10": + version "0.5.11" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.11.tgz#5bab8c660a6e23c13b2d23fcd1ee44a2db1b0cb7" + integrity sha512-YNlnKRWF2sVojTpIyzwou9XoTNbzbzONwRhOoniEioF1AtaitTvVZblaQRrAzChWQ1bLYyYSWzM18y4WwgzJ+A== + dependencies: + tslib "^2.4.0" + "@szmarczak/http-timer@^4.0.5": version "4.0.6" resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.6.tgz#b4a914bb62e7c272d4e5989fe4440f812ab1d807" @@ -301,6 +316,16 @@ "@types/node" "*" "@types/responselike" "*" +"@types/command-line-args@^5.2.3": + version "5.2.3" + resolved "https://registry.yarnpkg.com/@types/command-line-args/-/command-line-args-5.2.3.tgz#553ce2fd5acf160b448d307649b38ffc60d39639" + integrity sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw== + +"@types/command-line-usage@^5.0.4": + version "5.0.4" + resolved "https://registry.yarnpkg.com/@types/command-line-usage/-/command-line-usage-5.0.4.tgz#374e4c62d78fbc5a670a0f36da10235af879a0d5" + integrity sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg== + "@types/estree@*": version "0.0.50" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.50.tgz#1e0caa9364d3fccd2931c3ed96fdbeaa5d4cca83" @@ -328,6 +353,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.11.tgz#6ea7342dfb379ea1210835bada87b3c512120234" integrity sha512-KB0sixD67CeecHC33MYn+eYARkqTheIRNuu97y2XMjR7Wu3XibO1vaY6VBV6O/a89SPI81cEUIYT87UqUWlZNw== +"@types/node@^20.12.7": + version "20.12.12" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.12.tgz#7cbecdf902085cec634fdb362172dfe12b8f2050" + integrity sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw== + dependencies: + undici-types "~5.26.4" + "@types/parse-json@^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" @@ -466,6 +498,21 @@ anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" +apache-arrow@16: + version "16.1.0" + resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-16.1.0.tgz#7aa8d0d436dd0995d9dc5c36febf380d5b207209" + integrity sha512-G6GiM6tzPDdGnKUnVkvVr1Nt5+hUaCMBISiasMSiJwI5L5GKDv5Du7Avc2kxlFfB/LEK2LTqh2GKSxutMdf8vQ== + dependencies: + "@swc/helpers" "^0.5.10" + "@types/command-line-args" "^5.2.3" + "@types/command-line-usage" "^5.0.4" + "@types/node" "^20.12.7" + command-line-args "^5.2.1" + command-line-usage "^7.0.1" + flatbuffers "^24.3.25" + json-bignum "^0.0.3" + tslib "^2.6.2" + aproba@^1.0.3: version "1.2.0" resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" @@ -484,6 +531,16 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== +array-back@^3.0.1, array-back@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/array-back/-/array-back-3.1.0.tgz#b8859d7a508871c9a7b2cf42f99428f65e96bfb0" + integrity sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q== + +array-back@^6.2.2: + version "6.2.2" + resolved "https://registry.yarnpkg.com/array-back/-/array-back-6.2.2.tgz#f567d99e9af88a6d3d2f9dfcc21db6f9ba9fd157" + integrity sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw== + asap@^2.0.0: version "2.0.6" resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" @@ -684,6 +741,13 @@ caseless@~0.12.0: resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= +chalk-template@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/chalk-template/-/chalk-template-0.4.0.tgz#692c034d0ed62436b9062c1707fadcd0f753204b" + integrity sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg== + dependencies: + chalk "^4.1.2" + chalk@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -693,7 +757,7 @@ chalk@^2.0.0: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^4.0.0, chalk@^4.1.0: +chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -826,6 +890,26 @@ combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" +command-line-args@^5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/command-line-args/-/command-line-args-5.2.1.tgz#c44c32e437a57d7c51157696893c5909e9cec42e" + integrity sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg== + dependencies: + array-back "^3.1.0" + find-replace "^3.0.0" + lodash.camelcase "^4.3.0" + typical "^4.0.0" + +command-line-usage@^7.0.0, command-line-usage@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/command-line-usage/-/command-line-usage-7.0.1.tgz#e540afef4a4f3bc501b124ffde33956309100655" + integrity sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ== + dependencies: + array-back "^6.2.2" + chalk-template "^0.4.0" + table-layout "^3.0.0" + typical "^7.1.1" + commander@7: version "7.2.0" resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" @@ -1697,6 +1781,13 @@ find-cache-dir@^3.3.1: make-dir "^3.0.2" pkg-dir "^4.1.0" +find-replace@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/find-replace/-/find-replace-3.0.0.tgz#3e7e23d3b05167a76f770c9fbd5258b0def68c38" + integrity sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ== + dependencies: + array-back "^3.0.1" + find-up@5.0.0, find-up@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" @@ -1726,6 +1817,11 @@ flat@^5.0.2: resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== +flatbuffers@^24.3.25: + version "24.3.25" + resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-24.3.25.tgz#e2f92259ba8aa53acd0af7844afb7c7eb95e7089" + integrity sha512-3HDgPbgiwWMI9zVB7VYBHaMrbOO7Gm0v+yD2FV/sCKj+9NDeVL7BOBYUuhWAQGKWOzBo8S9WdMvV0eixO233XQ== + flatted@^3.1.0: version "3.2.4" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.4.tgz#28d9969ea90661b5134259f312ab6aa7929ac5e2" @@ -2335,6 +2431,11 @@ jsesc@3: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-3.0.2.tgz#bb8b09a6597ba426425f2e4a07245c3d00b9343e" integrity sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g== +json-bignum@^0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/json-bignum/-/json-bignum-0.0.3.tgz#41163b50436c773d82424dbc20ed70db7604b8d7" + integrity sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg== + json-buffer@3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" @@ -2463,6 +2564,11 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" +lodash.assignwith@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz#127a97f02adc41751a954d24b0de17e100e038eb" + integrity sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g== + lodash.camelcase@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" @@ -3667,6 +3773,11 @@ ssri@^8.0.0, ssri@^8.0.1: dependencies: minipass "^3.1.1" +stream-read-all@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/stream-read-all/-/stream-read-all-3.0.1.tgz#60762ae45e61d93ba0978cda7f3913790052ad96" + integrity sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A== + string-hash@^1.1.1: version "1.1.3" resolved "https://registry.yarnpkg.com/string-hash/-/string-hash-1.1.3.tgz#e8aafc0ac1855b4666929ed7dd1275df5d6c811b" @@ -3752,6 +3863,19 @@ symbol-tree@^3.2.4: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== +table-layout@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-3.0.2.tgz#69c2be44388a5139b48c59cf21e73b488021769a" + integrity sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw== + dependencies: + "@75lb/deep-merge" "^1.1.1" + array-back "^6.2.2" + command-line-args "^5.2.1" + command-line-usage "^7.0.0" + stream-read-all "^3.0.1" + typical "^7.1.1" + wordwrapjs "^5.1.0" + tar@^6.0.2, tar@^6.1.0: version "6.1.11" resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621" @@ -3819,6 +3943,11 @@ tslib@^2.2.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== +tslib@^2.4.0, tslib@^2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" @@ -3857,6 +3986,21 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" +typical@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/typical/-/typical-4.0.0.tgz#cbeaff3b9d7ae1e2bbfaf5a4e6f11eccfde94fc4" + integrity sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw== + +typical@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/typical/-/typical-7.1.1.tgz#ba177ab7ab103b78534463ffa4c0c9754523ac1f" + integrity sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA== + +undici-types@~5.26.4: + version "5.26.5" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" @@ -4003,6 +4147,11 @@ word-wrap@^1.2.3, word-wrap@~1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== +wordwrapjs@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-5.1.0.tgz#4c4d20446dcc670b14fa115ef4f8fd9947af2b3a" + integrity sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg== + workerpool@6.1.5: version "6.1.5" resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.1.5.tgz#0f7cf076b6215fd7e1da903ff6f22ddd1886b581" From f0b9a8081e29b1e3ad45b907ef69ac2eac05a32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Wed, 22 May 2024 15:35:32 +0200 Subject: [PATCH 2/4] Update src/table.js Co-authored-by: Fabian Iwand --- src/table.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/table.js b/src/table.js index dd6096c..197f447 100644 --- a/src/table.js +++ b/src/table.js @@ -356,7 +356,7 @@ function alignof(base = {}, data, columns) { function type(data, column) { // duck-type Arrow table date fields - if (String(data?.schema?.fields?.find?.(d => d?.name === column)).match(//)) return "date"; + if (String(data?.schema?.fields?.find?.(d => d?.name === column)).endsWith("") return "date"; for (const d of data) { if (d == null) continue; const value = d[column]; From 51341e75f32db70f21eb267b02e7a2973681bf31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Wed, 22 May 2024 15:38:19 +0200 Subject: [PATCH 3/4] Update src/table.js --- src/table.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/table.js b/src/table.js index 197f447..54d091f 100644 --- a/src/table.js +++ b/src/table.js @@ -356,7 +356,7 @@ function alignof(base = {}, data, columns) { function type(data, column) { // duck-type Arrow table date fields - if (String(data?.schema?.fields?.find?.(d => d?.name === column)).endsWith("") return "date"; + if (String(data?.schema?.fields?.find?.(d => d?.name === column)).endsWith("")) return "date"; for (const d of data) { if (d == null) continue; const value = d[column]; From b88074ee6fa81c27e360e76da193f4714a9551af Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Fri, 26 Jul 2024 11:19:56 -0400 Subject: [PATCH 4/4] use numeric types --- package.json | 2 +- src/table.js | 28 ++++++++++++++++++++++++++-- yarn.lock | 28 ++++++++++++++-------------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index c52360d..296eb9d 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,8 @@ "@rollup/plugin-json": "4", "@rollup/plugin-node-resolve": "13", "@rollup/plugin-replace": "3", + "apache-arrow": "17", "clean-css": "5", - "apache-arrow": "16", "d3": "7", "eslint": "8", "js-beautify": "1", diff --git a/src/table.js b/src/table.js index 54d091f..1f708d8 100644 --- a/src/table.js +++ b/src/table.js @@ -355,8 +355,7 @@ function alignof(base = {}, data, columns) { } function type(data, column) { - // duck-type Arrow table date fields - if (String(data?.schema?.fields?.find?.(d => d?.name === column)).endsWith("")) return "date"; + if (isArrowTable(data)) return getArrowType(data, column); for (const d of data) { if (d == null) continue; const value = d[column]; @@ -367,6 +366,31 @@ function type(data, column) { } } +// https://github.com/observablehq/stdlib/blob/746ca2e69135df6178e4f3a17244def35d8d6b20/src/arrow.js#L4-L16 +function isArrowTable(value) { + return ( + typeof value.getChild === "function" && + typeof value.toArray === "function" && + value.schema && + Array.isArray(value.schema.fields) + ); +} + +// https://github.com/apache/arrow/blob/89f9a0948961f6e94f1ef5e4f310b707d22a3c11/js/src/enum.ts#L140-L141 +function getArrowType(value, column) { + const field = value.schema.fields.find((d) => d.name === column); + switch (field?.type.typeId) { + case 8: // Date + case 10: // Timestamp + return field.type.unit === 1 ? "date" : "number"; // millisecond + case 2: // Int + case 3: // Float + case 7: // Decimal + case 9: // Time + return "number"; + } +} + function lengthof(data) { if (typeof data.length === "number") return data.length; // array or array-like if (typeof data.size === "number") return data.size; // map, set diff --git a/yarn.lock b/yarn.lock index 2c1c4e0..b542807 100644 --- a/yarn.lock +++ b/yarn.lock @@ -282,10 +282,10 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.2.0.tgz#667bfc6186ae7c9e0b45a08960c551437176e1ca" integrity sha512-VkE3KLBmJwcCaVARtQpfuKcKv8gcBmUubrfHGF84dXuuW6jgsRYxPtzcIhPyK9WAPpRt2/xY6zkD9MnRaJzSyw== -"@swc/helpers@^0.5.10": - version "0.5.11" - resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.11.tgz#5bab8c660a6e23c13b2d23fcd1ee44a2db1b0cb7" - integrity sha512-YNlnKRWF2sVojTpIyzwou9XoTNbzbzONwRhOoniEioF1AtaitTvVZblaQRrAzChWQ1bLYyYSWzM18y4WwgzJ+A== +"@swc/helpers@^0.5.11": + version "0.5.12" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.12.tgz#37aaca95284019eb5d2207101249435659709f4b" + integrity sha512-KMZNXiGibsW9kvZAO1Pam2JPTDBm+KSHMMHWdsyI/1DbIZjT2A6Gy3hblVXUMEDvUAKq+e0vL0X0o54owWji7g== dependencies: tslib "^2.4.0" @@ -353,10 +353,10 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.11.tgz#6ea7342dfb379ea1210835bada87b3c512120234" integrity sha512-KB0sixD67CeecHC33MYn+eYARkqTheIRNuu97y2XMjR7Wu3XibO1vaY6VBV6O/a89SPI81cEUIYT87UqUWlZNw== -"@types/node@^20.12.7": - version "20.12.12" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.12.tgz#7cbecdf902085cec634fdb362172dfe12b8f2050" - integrity sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw== +"@types/node@^20.13.0": + version "20.14.12" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.12.tgz#129d7c3a822cb49fc7ff661235f19cfefd422b49" + integrity sha512-r7wNXakLeSsGT0H1AU863vS2wa5wBOK4bWMjZz2wj+8nBx+m5PeIn0k8AloSLpRuiwdRQZwarZqHE4FNArPuJQ== dependencies: undici-types "~5.26.4" @@ -498,15 +498,15 @@ anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" -apache-arrow@16: - version "16.1.0" - resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-16.1.0.tgz#7aa8d0d436dd0995d9dc5c36febf380d5b207209" - integrity sha512-G6GiM6tzPDdGnKUnVkvVr1Nt5+hUaCMBISiasMSiJwI5L5GKDv5Du7Avc2kxlFfB/LEK2LTqh2GKSxutMdf8vQ== +apache-arrow@17: + version "17.0.0" + resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-17.0.0.tgz#73d98566c86352c9a0314c03890dbd7211073827" + integrity sha512-X0p7auzdnGuhYMVKYINdQssS4EcKec9TCXyez/qtJt32DrIMGbzqiaMiQ0X6fQlQpw8Fl0Qygcv4dfRAr5Gu9Q== dependencies: - "@swc/helpers" "^0.5.10" + "@swc/helpers" "^0.5.11" "@types/command-line-args" "^5.2.3" "@types/command-line-usage" "^5.0.4" - "@types/node" "^20.12.7" + "@types/node" "^20.13.0" command-line-args "^5.2.1" command-line-usage "^7.0.1" flatbuffers "^24.3.25"