Support endpoints that don't support range requests in asyncBufferFromUrl

src/utils.js

@@ -86,18 +86,42 @@ export async function asyncBufferFromUrl({ url, byteLength, requestInit }) {
   if (!url) throw new Error('missing url')
   // byte length from HEAD request
   byteLength ||= await byteLengthFromUrl(url, requestInit)
+
+  /**
+   * A promise for the whole buffer, if range requests are not supported.
+   * @type {Promise<ArrayBuffer>|undefined}
+   */
+  let buffer = undefined
   const init = requestInit || {}
+
   return {
     byteLength,
     async slice(start, end) {
-      // fetch byte range from url
+      if (buffer) {
+        return buffer.then(buffer => buffer.slice(start, end));
+      }
+
       const headers = new Headers(init.headers)
       const endStr = end === undefined ? '' : end - 1
       headers.set('Range', `bytes=${start}-${endStr}`)
+
       const res = await fetch(url, { ...init, headers })
       if (!res.ok || !res.body) throw new Error(`fetch failed ${res.status}`)
-      return res.arrayBuffer()
-    },
+
+      switch (res.status) {
+        // Endpoint does not support range requests and returned the whole object
+        case 200:
+          buffer = res.arrayBuffer();
+          return buffer.then(buffer => buffer.slice(start, end));
+
+        // The endpoint supports range requests and sent us the requested range
+        case 206:
+          return res.arrayBuffer();
+
+        default:
+          throw new Error(`fetch received unexpected status code ${res.status}`)
+      }
+    }
   }
 }
 

test/utils.test.js

@@ -1,5 +1,6 @@
 import { describe, expect, it, vi } from 'vitest'
 import { asyncBufferFromUrl, byteLengthFromUrl, toJson } from '../src/utils.js'
+import { arrayBuffer } from 'stream/consumers'
 
 describe('toJson', () => {
   it('convert undefined to null', () => {
@@ -114,6 +115,7 @@ describe('asyncBufferFromUrl', () => {
     global.fetch = vi.fn().mockResolvedValue({
       ok: true,
       body: {},
+      status: 206,
       arrayBuffer: () => Promise.resolve(mockArrayBuffer),
     })
 
@@ -131,6 +133,7 @@ describe('asyncBufferFromUrl', () => {
     global.fetch = vi.fn().mockResolvedValue({
       ok: true,
       body: {},
+      status: 206,
       arrayBuffer: () => Promise.resolve(mockArrayBuffer),
     })
 
@@ -191,6 +194,7 @@ describe('asyncBufferFromUrl', () => {
       return Promise.resolve({
         ok: true,
         body: {},
+        status: 206,
         arrayBuffer: () => Promise.resolve(mockArrayBuffer),
       })
     })
@@ -203,4 +207,42 @@ describe('asyncBufferFromUrl', () => {
 
     await expect(withHeaders.slice(0, 10)).rejects.toThrow('fetch failed 404')
   })
+
+  describe("when range requests are unsupported", () => {
+    it('creates an AsyncBuffer with the correct byte length', async () => {
+      const mockArrayBuffer = new ArrayBuffer(1024)
+      global.fetch = vi.fn().mockResolvedValue({
+        ok: true,
+        status: 200,
+        body: {},
+        arrayBuffer: () => Promise.resolve(mockArrayBuffer),
+      });
+
+      const buffer = await asyncBufferFromUrl({ url: 'https://example.com', byteLength: 1024 })
+      const chunk = await buffer.slice(0, 100);
+
+      expect(fetch).toHaveBeenCalledWith('https://example.com', {
+        headers: new Headers({ Range: 'bytes=0-99' })
+      });
+
+      expect(chunk.byteLength).toBe(100);
+    })
+
+    it('does not make multiple requests for multiple slices', async () => {
+      const mockArrayBuffer = new ArrayBuffer(1024)
+      global.fetch = vi.fn().mockResolvedValue({
+        ok: true,
+        status: 200,
+        body: {},
+        arrayBuffer: () => Promise.resolve(mockArrayBuffer)
+      });
+
+      const buffer = await asyncBufferFromUrl({ url: 'https://example.com', byteLength: 1024 })
+
+      await buffer.slice(0, 100)
+      await buffer.slice(550, 600)
+
+      expect(fetch).toBeCalledTimes(1)
+    })
+  })
 })