Enabling file streaming operations

Support for streaming is critical for applications that need to send or receive large amounts of data between client and server without first buffering the data into memory, potentially exhausting this system resource.

Streaming download

If you create an endpoint with a top-level binary response body, you can treat that response as a streamable, and iterate over it without loading the entire response into memory. This is useful for large file downloads, long-running streaming responses, and more.

In an OpenAPI document, this can be modeled as a binary stream. Here’s an example of a get operation with content type as application/octet-stream.

/streamable:
  get:
    operationId: streamable
    responses:
      "200":
        description: OK
        content:
          application/octet-stream:
            schema:
              title: bytes
              type: string
              format: binary

import fs from "node:fs";
import { Writable } from "node:stream";
import { SDK } from "@speakeasy/super-sdk";

async function run() {
  const sdk = new SDK();

  const result = await sdk.streamable("UR123");

  const destination = Writable.toWeb(
    fs.createWriteStream("./report.csv")
  );

  await result.data.pipeTo(destination);
}

run();

ctx := context.Background()
s := sdk.New()

file, err := os.Create("./report.csv")

if err != nil {
  // ... error handling ...
}

response, err := s.streamable(ctx)

if err != nil {
  // ... error handling ...
}

defer response.Data.Close()

_, err = io.Copy(file, response.Data)

Streaming uploads

Streaming is useful when uploading large files. Certain SDK methods will be generated that accept files as part of a multipart request. It is possible (and recommended) to upload files as a stream rather than reading the entire contents into memory. This avoids excessive memory consumption and potentially crashing with out-of-memory errors when working with large files.

In this example, a request to upload a file is managed as a multipart/form-data request.

/file:
  post:
    summary: Upload file
    operationId: upload
    requestBody:
      content:
        multipart/form-data:
          schema:
            $ref: "#/components/schemas/UploadFileRequest"
      required: true
    responses:
      "200":
        description: ""
        headers:
          Action-Id:
            required: true
            schema:
              type: string
        content:
          application/json:
            schema:
              $ref: "#/components/schemas/File"

import { openAsBlob } from "node:fs";

import { SDK } from "@speakeasy/super-sdk";

async function run() {
  const sdk = new SDK();

  const fileHandle = await openAsBlob("./src/sample.txt");

  const result = await sdk.upload({ file: fileHandle });

  console.log(result);
}
run();

ctx := context.Background()
s := sdk.New()

file, err := os.Open("./src/sample.txt")

if err != nil {
  // ... error handling ...
}

response, err := s.upload(ctx, file)