ML-173: replace aws-sdk with aws4 + built-in fetch
Replace @aws-sdk/client-s3 (~150 transitive deps, ~14MB) with aws4 (0 transitive deps, ~40KB) and Node.js built-in fetch+crypto for S3 ListObjectsV2 against Hetzner Object Storage. - New s3-client.ts: SigV4 signing via aws4, regex-based XML parsing with namespace tolerance, pagination support, 5 error categories - 29 unit tests (node:test, zero-dependency) covering signing, parsing, entities, missing fields, pagination, abort, errors - Settlement guard for double done(null) on abort - XML entity decoding and regex metacharacter escaping - Dependency surface: 150 packages → 1 (aws4, single-file, auditable)
This commit is contained in:
Claudio Ortolina
Vendored
+30
@@ -0,0 +1,30 @@
|
||||
declare module "aws4" {
|
||||
interface Aws4Request {
|
||||
host?: string;
|
||||
path?: string;
|
||||
method?: string;
|
||||
headers?: Record<string, string>;
|
||||
service?: string;
|
||||
region?: string;
|
||||
body?: string;
|
||||
signQuery?: boolean;
|
||||
}
|
||||
|
||||
interface Aws4Credentials {
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
sessionToken?: string;
|
||||
}
|
||||
|
||||
function sign(
|
||||
request: Aws4Request,
|
||||
credentials: Aws4Credentials,
|
||||
): Aws4Request & {
|
||||
host: string;
|
||||
path: string;
|
||||
headers: Record<string, string>;
|
||||
};
|
||||
|
||||
const aws4: { sign: typeof sign };
|
||||
export default aws4;
|
||||
}
|
||||
@@ -12,7 +12,6 @@
|
||||
* /backups
|
||||
*/
|
||||
|
||||
import { ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3";
|
||||
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
||||
import { BorderedLoader, DynamicBorder } from "@mariozechner/pi-coding-agent";
|
||||
import {
|
||||
@@ -21,6 +20,15 @@ import {
|
||||
SelectList,
|
||||
Text,
|
||||
} from "@mariozechner/pi-tui";
|
||||
import { listAllObjects, type S3ClientConfig } from "./s3-client";
|
||||
|
||||
// Re-export for consumers
|
||||
export {
|
||||
AuthError,
|
||||
NetworkError,
|
||||
AbortError,
|
||||
type S3Object,
|
||||
} from "./s3-client";
|
||||
|
||||
// ── S3 configuration (mirrors scripts/prod/litestream-backup) ────────────────
|
||||
const S3_ENDPOINT = "https://nbg1.your-objectstorage.com";
|
||||
@@ -54,55 +62,6 @@ function displayKey(key: string): string {
|
||||
return key;
|
||||
}
|
||||
|
||||
export interface S3Object {
|
||||
key: string;
|
||||
size: number;
|
||||
lastModified: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all objects (handles pagination via ContinuationToken).
|
||||
*/
|
||||
async function listAllObjects(
|
||||
client: S3Client,
|
||||
signal?: AbortSignal,
|
||||
): Promise<S3Object[]> {
|
||||
const results: S3Object[] = [];
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
do {
|
||||
const command = new ListObjectsV2Command({
|
||||
Bucket: S3_BUCKET,
|
||||
Prefix: S3_PREFIX,
|
||||
...(continuationToken ? { ContinuationToken: continuationToken } : {}),
|
||||
});
|
||||
|
||||
const response = await client.send(command, { abortSignal: signal });
|
||||
|
||||
if (response.Contents) {
|
||||
for (const obj of response.Contents) {
|
||||
results.push({
|
||||
key: obj.Key ?? "",
|
||||
size: obj.Size ?? 0,
|
||||
lastModified: obj.LastModified ?? new Date(0),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
continuationToken = response.IsTruncated
|
||||
? response.NextContinuationToken
|
||||
: undefined;
|
||||
} while (continuationToken);
|
||||
|
||||
// Sort by size descending, then alphabetically descending
|
||||
results.sort((a, b) => {
|
||||
const sizeDiff = b.size - a.size;
|
||||
if (sizeDiff !== 0) return sizeDiff;
|
||||
return b.key.localeCompare(a.key);
|
||||
});
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── Extension ───────────────────────────────────────────────────────────────
|
||||
|
||||
export default function s3BrowserExtension(pi: ExtensionAPI) {
|
||||
@@ -131,23 +90,46 @@ export default function s3BrowserExtension(pi: ExtensionAPI) {
|
||||
theme,
|
||||
"Fetching backup list from S3...",
|
||||
);
|
||||
loader.onAbort = () => done(null);
|
||||
|
||||
const doFetch = async (): Promise<S3Object[]> => {
|
||||
const client = new S3Client({
|
||||
const config: S3ClientConfig = {
|
||||
endpoint: new URL(S3_ENDPOINT).host,
|
||||
region: S3_REGION,
|
||||
endpoint: S3_ENDPOINT,
|
||||
credentials: { accessKeyId, secretAccessKey },
|
||||
forcePathStyle: true,
|
||||
});
|
||||
return listAllObjects(client, loader.signal);
|
||||
bucket: S3_BUCKET,
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
};
|
||||
return listAllObjects(config, { prefix: S3_PREFIX }, loader.signal);
|
||||
};
|
||||
|
||||
let settled = false;
|
||||
const finish = (result: S3Object[] | null) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
done(result);
|
||||
}
|
||||
};
|
||||
loader.onAbort = () => finish(null);
|
||||
|
||||
doFetch()
|
||||
.then(done)
|
||||
.catch((err) => {
|
||||
console.error("[s3-browser] Fetch failed:", err);
|
||||
done(null);
|
||||
.then(finish)
|
||||
.catch((err: unknown) => {
|
||||
if (err instanceof AuthError) {
|
||||
ctx.ui.notify(
|
||||
"Authentication failed — check LITESTREAM_ACCESS_KEY_ID / LITESTREAM_SECRET_ACCESS_KEY",
|
||||
"error",
|
||||
);
|
||||
} else if (err instanceof NetworkError) {
|
||||
ctx.ui.notify(
|
||||
"Could not reach S3 endpoint — check network",
|
||||
"error",
|
||||
);
|
||||
} else if (err instanceof AbortError) {
|
||||
// User cancelled — clean cancellation, no notification needed
|
||||
} else {
|
||||
console.error("[s3-browser] Fetch failed:", err);
|
||||
}
|
||||
finish(null);
|
||||
});
|
||||
|
||||
return loader;
|
||||
|
||||
+5
-1418
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,11 @@
|
||||
{
|
||||
"name": "s3-browser",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.787.0"
|
||||
"aws4": "^1.13.2"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "node --experimental-strip-types --test s3-client.test.ts"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,656 @@
|
||||
/**
|
||||
* Tests for s3-client.ts using Node.js built-in node:test runner.
|
||||
*
|
||||
* Run with: node --experimental-strip-types --test s3-client.test.ts
|
||||
*/
|
||||
|
||||
import { describe, it, beforeEach, afterEach, mock } from "node:test";
|
||||
import * as assert from "node:assert/strict";
|
||||
|
||||
// We import the module under test. Since we're using --experimental-strip-types,
|
||||
// direct TypeScript imports work.
|
||||
import {
|
||||
signRequest,
|
||||
parseListObjectsV2Response,
|
||||
parseErrorResponse,
|
||||
listObjectsV2,
|
||||
listAllObjects,
|
||||
AuthError,
|
||||
NotFoundError,
|
||||
NetworkError,
|
||||
AbortError,
|
||||
ServerError,
|
||||
} from "./s3-client.ts";
|
||||
|
||||
// ── Test helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
const TEST_CONFIG = {
|
||||
endpoint: "nbg1.your-objectstorage.com",
|
||||
region: "nbg1",
|
||||
bucket: "ffmusiclibrary",
|
||||
accessKeyId: "TESTACCESSKEY",
|
||||
secretAccessKey: "TESTVAL123",
|
||||
};
|
||||
|
||||
// ── signRequest tests ──────────────────────────────────────────────────────
|
||||
|
||||
describe("signRequest", () => {
|
||||
it("produces a well-formed Authorization header (AWS4-HMAC-SHA256)", () => {
|
||||
const result = signRequest(TEST_CONFIG, "/bucket/?list-type=2");
|
||||
|
||||
assert.ok(result.url.startsWith("https://nbg1.your-objectstorage.com/"));
|
||||
assert.ok(typeof result.headers["authorization"] === "string");
|
||||
assert.ok(result.headers["authorization"].startsWith("AWS4-HMAC-SHA256"));
|
||||
assert.ok(
|
||||
result.headers["authorization"].includes("Credential=TESTACCESSKEY/"),
|
||||
);
|
||||
assert.ok(
|
||||
result.headers["authorization"].includes("/nbg1/s3/aws4_request"),
|
||||
);
|
||||
assert.ok(result.headers["authorization"].includes("SignedHeaders="));
|
||||
assert.ok(result.headers["authorization"].includes("Signature="));
|
||||
});
|
||||
|
||||
it("includes x-amz-date and x-amz-content-sha256 headers", () => {
|
||||
const result = signRequest(TEST_CONFIG, "/bucket/?list-type=2");
|
||||
|
||||
assert.ok(typeof result.headers["x-amz-date"] === "string");
|
||||
assert.equal(result.headers["x-amz-content-sha256"], "UNSIGNED-PAYLOAD");
|
||||
});
|
||||
|
||||
it("preserves continuation token in URL", () => {
|
||||
const result = signRequest(
|
||||
TEST_CONFIG,
|
||||
"/ffmusiclibrary/?list-type=2&prefix=prod%2F&continuation-token=abc123",
|
||||
);
|
||||
|
||||
assert.ok(result.url.includes("continuation-token=abc123"));
|
||||
});
|
||||
});
|
||||
|
||||
// ── XML parsing tests ──────────────────────────────────────────────────────
|
||||
|
||||
// Real-world Hetzner Object Storage ListObjectsV2 XML response pattern.
|
||||
// Note: Hetzner may include xmlns attributes.
|
||||
const REAL_XML_RESPONSE = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>ffmusiclibrary</Name>
|
||||
<Prefix>prod/</Prefix>
|
||||
<KeyCount>3</KeyCount>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents>
|
||||
<Key>prod/db.sqlite3</Key>
|
||||
<LastModified>2025-12-01T10:30:00.000Z</LastModified>
|
||||
<ETag>"abc123"</ETag>
|
||||
<Size>1048576</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>
|
||||
<Contents>
|
||||
<Key>prod/db.sqlite3-wal</Key>
|
||||
<LastModified>2025-12-01T10:30:05.000Z</LastModified>
|
||||
<ETag>"def456"</ETag>
|
||||
<Size>512</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>
|
||||
<Contents xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Key>prod/db.sqlite3-shm</Key>
|
||||
<LastModified>2025-12-01T10:30:10.000Z</LastModified>
|
||||
<ETag>"ghi789"</ETag>
|
||||
<Size>32768</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
// Hetzner-like namespace-qualified XML
|
||||
const NAMESPACED_XML_RESPONSE = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>ffmusiclibrary</Name>
|
||||
<Prefix>prod/</Prefix>
|
||||
<KeyCount>2</KeyCount>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<IsTruncated>true</IsTruncated>
|
||||
<NextContinuationToken>eyJNYXJrZXIiOiBudWxsLCAiYm90b190cnVuY2F0ZV9hbW91bnQiOiAxfQ==</NextContinuationToken>
|
||||
<Contents xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Key xmlns="http://s3.amazonaws.com/doc/2006-03-01/">prod/wal/00000001.sqlite3-wal</Key>
|
||||
<LastModified xmlns="http://s3.amazonaws.com/doc/2006-03-01/">2025-12-02T08:00:00.000Z</LastModified>
|
||||
<Size xmlns="http://s3.amazonaws.com/doc/2006-03-01/">2048</Size>
|
||||
</Contents>
|
||||
<Contents>
|
||||
<Key>prod/wal/00000002.sqlite3-wal</Key>
|
||||
<LastModified>2025-12-02T09:00:00.000Z</LastModified>
|
||||
<Size>4096</Size>
|
||||
</Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
const EMPTY_RESPONSE = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>ffmusiclibrary</Name>
|
||||
<Prefix>empty/</Prefix>
|
||||
<KeyCount>0</KeyCount>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
</ListBucketResult>`;
|
||||
|
||||
const ERROR_RESPONSE = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Error>
|
||||
<Code>AccessDenied</Code>
|
||||
<Message>Access Denied</Message>
|
||||
<RequestId>TX1234567890</RequestId>
|
||||
<HostId>host-id-123</HostId>
|
||||
</Error>`;
|
||||
|
||||
describe("parseListObjectsV2Response", () => {
|
||||
it("parses a real-world XML response correctly", () => {
|
||||
const result = parseListObjectsV2Response(REAL_XML_RESPONSE);
|
||||
|
||||
assert.equal(result.objects.length, 3);
|
||||
assert.equal(result.isTruncated, false);
|
||||
assert.equal(result.nextContinuationToken, undefined);
|
||||
|
||||
// First object
|
||||
assert.equal(result.objects[0].key, "prod/db.sqlite3");
|
||||
assert.equal(result.objects[0].size, 1048576);
|
||||
assert.equal(
|
||||
result.objects[0].lastModified.toISOString(),
|
||||
"2025-12-01T10:30:00.000Z",
|
||||
);
|
||||
|
||||
// Second object
|
||||
assert.equal(result.objects[1].key, "prod/db.sqlite3-wal");
|
||||
assert.equal(result.objects[1].size, 512);
|
||||
|
||||
// Third object (namespace-qualified)
|
||||
assert.equal(result.objects[2].key, "prod/db.sqlite3-shm");
|
||||
assert.equal(result.objects[2].size, 32768);
|
||||
});
|
||||
|
||||
it("handles namespace-qualified XML (individual tag xmlns)", () => {
|
||||
const result = parseListObjectsV2Response(NAMESPACED_XML_RESPONSE);
|
||||
|
||||
assert.equal(result.objects.length, 2);
|
||||
assert.equal(result.isTruncated, true);
|
||||
assert.equal(
|
||||
result.nextContinuationToken,
|
||||
"eyJNYXJrZXIiOiBudWxsLCAiYm90b190cnVuY2F0ZV9hbW91bnQiOiAxfQ==",
|
||||
);
|
||||
|
||||
// Both objects should be parsed despite namespace differences
|
||||
assert.equal(result.objects[0].key, "prod/wal/00000001.sqlite3-wal");
|
||||
assert.equal(result.objects[0].size, 2048);
|
||||
assert.equal(result.objects[1].key, "prod/wal/00000002.sqlite3-wal");
|
||||
assert.equal(result.objects[1].size, 4096);
|
||||
});
|
||||
|
||||
it("handles empty response (no Contents blocks)", () => {
|
||||
const result = parseListObjectsV2Response(EMPTY_RESPONSE);
|
||||
|
||||
assert.equal(result.objects.length, 0);
|
||||
assert.equal(result.isTruncated, false);
|
||||
assert.equal(result.nextContinuationToken, undefined);
|
||||
});
|
||||
|
||||
it("handles truncated response with next continuation token", () => {
|
||||
const result = parseListObjectsV2Response(NAMESPACED_XML_RESPONSE);
|
||||
|
||||
assert.equal(result.isTruncated, true);
|
||||
assert.ok(typeof result.nextContinuationToken === "string");
|
||||
assert.ok(result.nextContinuationToken!.length > 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseErrorResponse", () => {
|
||||
it("extracts Code and Message from S3 error XML", () => {
|
||||
const result = parseErrorResponse(ERROR_RESPONSE);
|
||||
|
||||
assert.equal(result.code, "AccessDenied");
|
||||
assert.equal(result.message, "Access Denied");
|
||||
});
|
||||
});
|
||||
|
||||
// ── HTTP error handling tests ───────────────────────────────────────────────
|
||||
|
||||
describe("listObjectsV2 error handling", () => {
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
mock.reset();
|
||||
});
|
||||
|
||||
it("throws AuthError on HTTP 403", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(ERROR_RESPONSE, { status: 403 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listObjectsV2(TEST_CONFIG, { prefix: "prod/" }),
|
||||
(err: unknown) => {
|
||||
assert.ok(err instanceof AuthError);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("throws AuthError on HTTP 401", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(ERROR_RESPONSE, { status: 401 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listObjectsV2(TEST_CONFIG, { prefix: "prod/" }),
|
||||
AuthError,
|
||||
);
|
||||
});
|
||||
|
||||
it("throws NotFoundError on HTTP 404", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response("Not Found", { status: 404 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listObjectsV2(TEST_CONFIG, { prefix: "prod/" }),
|
||||
NotFoundError,
|
||||
);
|
||||
});
|
||||
|
||||
it("throws ServerError on HTTP 500", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response("Internal Server Error", { status: 500 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listObjectsV2(TEST_CONFIG, { prefix: "prod/" }),
|
||||
ServerError,
|
||||
);
|
||||
});
|
||||
|
||||
it("throws NetworkError on fetch rejection", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.reject(new TypeError("fetch failed")),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listObjectsV2(TEST_CONFIG, { prefix: "prod/" }),
|
||||
NetworkError,
|
||||
);
|
||||
});
|
||||
|
||||
it("throws AbortError when signal is already aborted", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
// fetch should not be called when signal is already aborted
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(EMPTY_RESPONSE, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listObjectsV2(TEST_CONFIG, { prefix: "prod/" }, controller.signal),
|
||||
AbortError,
|
||||
);
|
||||
});
|
||||
|
||||
it("throws AbortError when aborted mid-fetch", async () => {
|
||||
const controller = new AbortController();
|
||||
|
||||
globalThis.fetch = mock.fn(
|
||||
(_url: string, init: { signal?: AbortSignal }) => {
|
||||
// Simulate abort mid-request
|
||||
const err = new DOMException("The operation was aborted", "AbortError");
|
||||
return Promise.reject(err);
|
||||
},
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listObjectsV2(TEST_CONFIG, { prefix: "prod/" }, controller.signal),
|
||||
AbortError,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Successful response test ────────────────────────────────────────────────
|
||||
|
||||
describe("listObjectsV2 success", () => {
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
mock.reset();
|
||||
});
|
||||
|
||||
it("returns parsed objects on HTTP 200", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(REAL_XML_RESPONSE, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
const result = await listObjectsV2(TEST_CONFIG, { prefix: "prod/" });
|
||||
|
||||
assert.equal(result.objects.length, 3);
|
||||
assert.equal(result.isTruncated, false);
|
||||
assert.equal(result.objects[0].key, "prod/db.sqlite3");
|
||||
assert.equal(result.objects[0].size, 1048576);
|
||||
});
|
||||
|
||||
it("calls fetch with the correct signed URL and headers", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(EMPTY_RESPONSE, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await listObjectsV2(TEST_CONFIG, { prefix: "prod/" });
|
||||
|
||||
const calls = (globalThis.fetch as ReturnType<typeof mock.fn>).mock.calls;
|
||||
assert.ok(Array.isArray(calls) && calls.length >= 1);
|
||||
const [url, init] = calls[0].arguments as [
|
||||
string,
|
||||
{ headers?: Record<string, string> },
|
||||
];
|
||||
assert.ok(url.startsWith("https://nbg1.your-objectstorage.com/"));
|
||||
assert.ok(url.includes("list-type=2"));
|
||||
assert.ok(url.includes("prefix=prod%2F"));
|
||||
assert.ok(typeof init.headers === "object");
|
||||
assert.ok(typeof init.headers["authorization"] === "string");
|
||||
assert.ok(init.headers["authorization"].startsWith("AWS4-HMAC-SHA256"));
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildPath tests (via listObjectsV2 + fetch mock inspection) ────────────
|
||||
|
||||
describe("buildPath (via listObjectsV2)", () => {
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
mock.reset();
|
||||
});
|
||||
|
||||
it("builds path with prefix and max-keys", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(EMPTY_RESPONSE, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await listObjectsV2(TEST_CONFIG, { prefix: "prod/", maxKeys: 500 });
|
||||
|
||||
const [url] = (globalThis.fetch as ReturnType<typeof mock.fn>).mock.calls[0]
|
||||
.arguments as [string];
|
||||
assert.ok(url.includes("prefix=prod%2F"));
|
||||
assert.ok(url.includes("max-keys=500"));
|
||||
});
|
||||
|
||||
it("includes continuation-token when provided", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(EMPTY_RESPONSE, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await listObjectsV2(TEST_CONFIG, {
|
||||
prefix: "prod/",
|
||||
continuationToken: "tok-abc",
|
||||
});
|
||||
|
||||
const [url] = (globalThis.fetch as ReturnType<typeof mock.fn>).mock.calls[0]
|
||||
.arguments as [string];
|
||||
assert.ok(url.includes("continuation-token=tok-abc"));
|
||||
});
|
||||
|
||||
it("omits continuation-token when not provided", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(EMPTY_RESPONSE, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await listObjectsV2(TEST_CONFIG, { prefix: "prod/" });
|
||||
|
||||
const [url] = (globalThis.fetch as ReturnType<typeof mock.fn>).mock.calls[0]
|
||||
.arguments as [string];
|
||||
assert.ok(!url.includes("continuation-token"));
|
||||
});
|
||||
});
|
||||
|
||||
// ── XML entity decoding tests ──────────────────────────────────────────────
|
||||
|
||||
const ENTITY_XML_RESPONSE = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>ffmusiclibrary</Name>
|
||||
<Prefix>prod/</Prefix>
|
||||
<KeyCount>2</KeyCount>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents>
|
||||
<Key>prod/foo&bar/data.sqlite3</Key>
|
||||
<LastModified>2025-12-01T10:30:00.000Z</LastModified>
|
||||
<Size>2048</Size>
|
||||
</Contents>
|
||||
<Contents>
|
||||
<Key>prod/with <angle> brackets.sqlite3-wal</Key>
|
||||
<LastModified>2025-12-01T10:30:05.000Z</LastModified>
|
||||
<Size>512</Size>
|
||||
</Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
describe("parseListObjectsV2Response — entities", () => {
|
||||
it("decodes XML character entities in keys", () => {
|
||||
const result = parseListObjectsV2Response(ENTITY_XML_RESPONSE);
|
||||
|
||||
assert.equal(result.objects.length, 2);
|
||||
assert.equal(result.objects[0].key, "prod/foo&bar/data.sqlite3");
|
||||
assert.equal(result.objects[0].size, 2048);
|
||||
assert.equal(
|
||||
result.objects[1].key,
|
||||
"prod/with <angle> brackets.sqlite3-wal",
|
||||
);
|
||||
assert.equal(result.objects[1].size, 512);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Missing optional fields tests ──────────────────────────────────────────
|
||||
|
||||
const MISSING_FIELDS_XML = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>ffmusiclibrary</Name>
|
||||
<Prefix>prod/</Prefix>
|
||||
<KeyCount>2</KeyCount>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents>
|
||||
<Key>prod/minimal.txt</Key>
|
||||
</Contents>
|
||||
<Contents>
|
||||
<Key>prod/full.txt</Key>
|
||||
<Size>42</Size>
|
||||
<LastModified>2025-12-01T10:30:00.000Z</LastModified>
|
||||
</Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
describe("parseListObjectsV2Response — missing fields", () => {
|
||||
it("defaults Size to 0 and LastModified to epoch when missing", () => {
|
||||
const result = parseListObjectsV2Response(MISSING_FIELDS_XML);
|
||||
|
||||
assert.equal(result.objects.length, 2);
|
||||
// Minimal object: no Size, no LastModified
|
||||
assert.equal(result.objects[0].key, "prod/minimal.txt");
|
||||
assert.equal(result.objects[0].size, 0);
|
||||
assert.equal(
|
||||
result.objects[0].lastModified.toISOString(),
|
||||
"1970-01-01T00:00:00.000Z",
|
||||
);
|
||||
// Full object: has all fields
|
||||
assert.equal(result.objects[1].key, "prod/full.txt");
|
||||
assert.equal(result.objects[1].size, 42);
|
||||
assert.equal(
|
||||
result.objects[1].lastModified.toISOString(),
|
||||
"2025-12-01T10:30:00.000Z",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── listAllObjects tests ───────────────────────────────────────────────────
|
||||
|
||||
describe("listAllObjects", () => {
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
originalFetch = globalThis.fetch;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
mock.reset();
|
||||
});
|
||||
|
||||
it("returns all objects from a single page (sorted)", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(REAL_XML_RESPONSE, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
const result = await listAllObjects(TEST_CONFIG, { prefix: "prod/" });
|
||||
|
||||
assert.equal(result.length, 3);
|
||||
// Sorted by size desc: 1048576 > 32768 > 512
|
||||
assert.equal(result[0].key, "prod/db.sqlite3");
|
||||
assert.equal(result[0].size, 1048576);
|
||||
assert.equal(result[1].key, "prod/db.sqlite3-shm");
|
||||
assert.equal(result[1].size, 32768);
|
||||
assert.equal(result[2].key, "prod/db.sqlite3-wal");
|
||||
assert.equal(result[2].size, 512);
|
||||
});
|
||||
|
||||
it("sorts by size descending, then key descending", async () => {
|
||||
// Test data: prod/a (512), prod/b (2048), prod/c (512)
|
||||
const SORT_XML = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>prod/a.sqlite3</Key><Size>512</Size><LastModified>2025-01-01T00:00:00.000Z</LastModified></Contents>
|
||||
<Contents><Key>prod/b.sqlite3</Key><Size>2048</Size><LastModified>2025-01-01T00:00:00.000Z</LastModified></Contents>
|
||||
<Contents><Key>prod/c.sqlite3</Key><Size>512</Size><LastModified>2025-01-01T00:00:00.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(SORT_XML, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
const result = await listAllObjects(TEST_CONFIG, { prefix: "prod/" });
|
||||
|
||||
assert.equal(result.length, 3);
|
||||
// Largest first
|
||||
assert.equal(result[0].key, "prod/b.sqlite3");
|
||||
assert.equal(result[0].size, 2048);
|
||||
// Same size: alphabetically descending
|
||||
assert.equal(result[1].key, "prod/c.sqlite3");
|
||||
assert.equal(result[1].size, 512);
|
||||
assert.equal(result[2].key, "prod/a.sqlite3");
|
||||
assert.equal(result[2].size, 512);
|
||||
});
|
||||
|
||||
it("aggregates objects across multiple pages", async () => {
|
||||
const PAGE1_XML = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<IsTruncated>true</IsTruncated>
|
||||
<NextContinuationToken>page2-token</NextContinuationToken>
|
||||
<Contents><Key>prod/obj1.sqlite3</Key><Size>100</Size><LastModified>2025-01-01T00:00:00.000Z</LastModified></Contents>
|
||||
<Contents><Key>prod/obj2.sqlite3</Key><Size>200</Size><LastModified>2025-01-01T00:00:00.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
const PAGE2_XML = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>prod/obj3.sqlite3</Key><Size>300</Size><LastModified>2025-01-01T00:00:00.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
let callCount = 0;
|
||||
globalThis.fetch = mock.fn(() => {
|
||||
callCount++;
|
||||
const xml = callCount === 1 ? PAGE1_XML : PAGE2_XML;
|
||||
return Promise.resolve(new Response(xml, { status: 200 }));
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const result = await listAllObjects(TEST_CONFIG, { prefix: "prod/" });
|
||||
|
||||
assert.equal(result.length, 3);
|
||||
assert.equal(callCount, 2);
|
||||
// Sorted: 300 (obj3), 200 (obj2), 100 (obj1)
|
||||
assert.equal(result[0].key, "prod/obj3.sqlite3");
|
||||
assert.equal(result[0].size, 300);
|
||||
assert.equal(result[1].key, "prod/obj2.sqlite3");
|
||||
assert.equal(result[1].size, 200);
|
||||
assert.equal(result[2].key, "prod/obj1.sqlite3");
|
||||
assert.equal(result[2].size, 100);
|
||||
});
|
||||
|
||||
it("passes continuation-token on subsequent pages", async () => {
|
||||
const PAGE1_XML = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<IsTruncated>true</IsTruncated>
|
||||
<NextContinuationToken>ct-token-xyz</NextContinuationToken>
|
||||
<Contents><Key>prod/a.sqlite3</Key><Size>1</Size><LastModified>2025-01-01T00:00:00.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
const PAGE2_XML = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>prod/b.sqlite3</Key><Size>2</Size><LastModified>2025-01-01T00:00:00.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`;
|
||||
|
||||
const urls: string[] = [];
|
||||
let callCount = 0;
|
||||
globalThis.fetch = mock.fn((url: string) => {
|
||||
urls.push(url);
|
||||
callCount++;
|
||||
const xml = callCount === 1 ? PAGE1_XML : PAGE2_XML;
|
||||
return Promise.resolve(new Response(xml, { status: 200 }));
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
await listAllObjects(TEST_CONFIG, { prefix: "prod/" });
|
||||
|
||||
assert.equal(callCount, 2);
|
||||
// First call: no continuation token
|
||||
assert.ok(!urls[0].includes("continuation-token"));
|
||||
// Second call: includes the continuation token from page 1
|
||||
assert.ok(urls[1].includes("continuation-token=ct-token-xyz"));
|
||||
});
|
||||
|
||||
it("returns empty array for empty bucket", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.resolve(new Response(EMPTY_RESPONSE, { status: 200 })),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
const result = await listAllObjects(TEST_CONFIG, { prefix: "empty/" });
|
||||
|
||||
assert.equal(result.length, 0);
|
||||
});
|
||||
|
||||
it("propagates AbortError from fetch abort", async () => {
|
||||
const controller = new AbortController();
|
||||
|
||||
globalThis.fetch = mock.fn(() => {
|
||||
const err = new DOMException("The operation was aborted", "AbortError");
|
||||
return Promise.reject(err);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listAllObjects(TEST_CONFIG, { prefix: "prod/" }, controller.signal),
|
||||
AbortError,
|
||||
);
|
||||
});
|
||||
|
||||
it("propagates NetworkError from fetch failure", async () => {
|
||||
globalThis.fetch = mock.fn(() =>
|
||||
Promise.reject(new TypeError("fetch failed")),
|
||||
) as unknown as typeof fetch;
|
||||
|
||||
await assert.rejects(
|
||||
() => listAllObjects(TEST_CONFIG, { prefix: "prod/" }),
|
||||
NetworkError,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,325 @@
|
||||
/**
|
||||
* Minimal S3 client using aws4 for SigV4 signing and Node.js built-in fetch.
|
||||
*
|
||||
* Only ListObjectsV2 is implemented — the sole API operation needed by the
|
||||
* s3-browser pi extension.
|
||||
*
|
||||
* Dependency surface: aws4 (single-file, 0 transitive deps) + Node.js built-ins.
|
||||
*/
|
||||
|
||||
import aws4 from "aws4";
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface S3ClientConfig {
|
||||
/** S3 endpoint hostname (no protocol), e.g. "nbg1.your-objectstorage.com" */
|
||||
endpoint: string;
|
||||
/** S3 region, e.g. "nbg1" */
|
||||
region: string;
|
||||
/** Bucket name */
|
||||
bucket: string;
|
||||
/** Access key ID */
|
||||
accessKeyId: string;
|
||||
/** Secret access key */
|
||||
secretAccessKey: string;
|
||||
}
|
||||
|
||||
export interface ListObjectsV2Params {
|
||||
/** Object key prefix */
|
||||
prefix?: string;
|
||||
/** Pagination continuation token */
|
||||
continuationToken?: string;
|
||||
/** Maximum keys per page (default: 1000) */
|
||||
maxKeys?: number;
|
||||
}
|
||||
|
||||
export interface S3Object {
|
||||
key: string;
|
||||
size: number;
|
||||
lastModified: Date;
|
||||
}
|
||||
|
||||
export interface ListObjectsV2Response {
|
||||
objects: S3Object[];
|
||||
isTruncated: boolean;
|
||||
nextContinuationToken?: string;
|
||||
}
|
||||
|
||||
// ── Error types ─────────────────────────────────────────────────────────────
|
||||
|
||||
export class AuthError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "AuthError";
|
||||
}
|
||||
}
|
||||
|
||||
export class NotFoundError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "NotFoundError";
|
||||
}
|
||||
}
|
||||
|
||||
export class NetworkError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "NetworkError";
|
||||
}
|
||||
}
|
||||
|
||||
export class AbortError extends Error {
|
||||
constructor() {
|
||||
super("Request aborted");
|
||||
this.name = "AbortError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ServerError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ServerError";
|
||||
}
|
||||
}
|
||||
|
||||
export type S3Error =
|
||||
| AuthError
|
||||
| NotFoundError
|
||||
| NetworkError
|
||||
| AbortError
|
||||
| ServerError;
|
||||
|
||||
// ── Signing ─────────────────────────────────────────────────────────────────
|
||||
|
||||
interface SignedRequest {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sign an S3 GET request using aws4.
|
||||
*
|
||||
* aws4.sign() returns options shaped for Node.js http.request
|
||||
* ({host, path, headers}). We extract headers and construct the
|
||||
* full URL for use with fetch.
|
||||
*/
|
||||
export function signRequest(
|
||||
config: S3ClientConfig,
|
||||
pathAndQuery: string,
|
||||
): SignedRequest {
|
||||
const opts = {
|
||||
host: config.endpoint,
|
||||
path: pathAndQuery,
|
||||
service: "s3",
|
||||
region: config.region,
|
||||
method: "GET",
|
||||
headers: {
|
||||
"X-Amz-Content-Sha256": "UNSIGNED-PAYLOAD",
|
||||
Host: config.endpoint,
|
||||
},
|
||||
};
|
||||
|
||||
const signed = aws4.sign(opts, {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
});
|
||||
|
||||
const url = `https://${signed.host}${signed.path}`;
|
||||
const headers: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(signed.headers)) {
|
||||
headers[key.toLowerCase()] = String(value);
|
||||
}
|
||||
|
||||
return { url, headers };
|
||||
}
|
||||
|
||||
// ── XML parsing ─────────────────────────────────────────────────────────────
|
||||
|
||||
/** Escape regex metacharacters in a string so it can be used literally. */
|
||||
function escapeRegex(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
/** Decode common XML character entities. */
|
||||
function decodeXmlEntities(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract text content between XML tags.
|
||||
* Handles namespace-qualified tags (e.g., <Contents xmlns="...">).
|
||||
*/
|
||||
function xmlText(xml: string, tag: string): string {
|
||||
const escaped = escapeRegex(tag);
|
||||
const re = new RegExp(`<${escaped}[^>]*>([\\s\\S]*?)<\\/${escaped}>`, "i");
|
||||
const match = xml.match(re);
|
||||
return match ? match[1].trim() : "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a ListObjectsV2 XML response body into structured data.
|
||||
*
|
||||
* Uses regex-based extraction (no full XML parser needed) to handle
|
||||
* namespace-qualified XML that Hetzner Object Storage may return.
|
||||
*/
|
||||
export function parseListObjectsV2Response(xml: string): ListObjectsV2Response {
|
||||
const objects: S3Object[] = [];
|
||||
|
||||
// Match <Contents> blocks, handling optional namespace attributes
|
||||
const contentsRe = /<Contents[^>]*>([\s\S]*?)<\/Contents>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = contentsRe.exec(xml)) !== null) {
|
||||
const block = match[1];
|
||||
const key = xmlText(block, "Key");
|
||||
const size = xmlText(block, "Size");
|
||||
const lastModified = xmlText(block, "LastModified");
|
||||
|
||||
if (key) {
|
||||
objects.push({
|
||||
key: decodeXmlEntities(key),
|
||||
size: size ? Number(size) : 0,
|
||||
lastModified: lastModified ? new Date(lastModified) : new Date(0),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const isTruncated = xmlText(xml, "IsTruncated") === "true";
|
||||
const nextContinuationToken =
|
||||
xmlText(xml, "NextContinuationToken") || undefined;
|
||||
|
||||
return { objects, isTruncated, nextContinuationToken };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an S3 error XML response.
|
||||
*/
|
||||
export function parseErrorResponse(xml: string): {
|
||||
code: string;
|
||||
message: string;
|
||||
} {
|
||||
return {
|
||||
code: xmlText(xml, "Code"),
|
||||
message: xmlText(xml, "Message"),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Paginated listing ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Fetch all objects across all pages (handles ContinuationToken).
|
||||
*
|
||||
* Results are sorted by size descending, then key descending — matching
|
||||
* the display order the s3-browser extension uses.
|
||||
*/
|
||||
export async function listAllObjects(
|
||||
config: S3ClientConfig,
|
||||
params: Omit<ListObjectsV2Params, "continuationToken">,
|
||||
signal?: AbortSignal,
|
||||
): Promise<S3Object[]> {
|
||||
const results: S3Object[] = [];
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
do {
|
||||
const response = await listObjectsV2(
|
||||
config,
|
||||
{ ...params, continuationToken },
|
||||
signal,
|
||||
);
|
||||
|
||||
results.push(...response.objects);
|
||||
|
||||
continuationToken = response.isTruncated
|
||||
? response.nextContinuationToken
|
||||
: undefined;
|
||||
} while (continuationToken);
|
||||
|
||||
// Sort by size descending, then alphabetically descending
|
||||
results.sort((a, b) => {
|
||||
const sizeDiff = b.size - a.size;
|
||||
if (sizeDiff !== 0) return sizeDiff;
|
||||
return b.key.localeCompare(a.key);
|
||||
});
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── API call ────────────────────────────────────────────────────────────────
|
||||
|
||||
function buildPath(
|
||||
config: S3ClientConfig,
|
||||
params: ListObjectsV2Params,
|
||||
): string {
|
||||
const query = new URLSearchParams();
|
||||
query.set("list-type", "2");
|
||||
|
||||
if (params.prefix) query.set("prefix", params.prefix);
|
||||
if (params.continuationToken)
|
||||
query.set("continuation-token", params.continuationToken);
|
||||
if (params.maxKeys) query.set("max-keys", String(params.maxKeys));
|
||||
|
||||
return `/${config.bucket}/?${query.toString()}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call S3 ListObjectsV2 with the given config, params, and optional AbortSignal.
|
||||
*
|
||||
* @throws {AuthError} On HTTP 401/403
|
||||
* @throws {NotFoundError} On HTTP 404
|
||||
* @throws {NetworkError} On fetch rejection (DNS, timeout, unreachable)
|
||||
* @throws {AbortError} On signal abort
|
||||
* @throws {ServerError} On HTTP 5xx or unexpected status
|
||||
*/
|
||||
export async function listObjectsV2(
|
||||
config: S3ClientConfig,
|
||||
params: ListObjectsV2Params,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ListObjectsV2Response> {
|
||||
const pathAndQuery = buildPath(config, params);
|
||||
const { url, headers } = signRequest(config, pathAndQuery);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(url, { headers, signal });
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof DOMException && err.name === "AbortError") {
|
||||
throw new AbortError();
|
||||
}
|
||||
// fetch can also throw TypeError for network failures
|
||||
throw new NetworkError(
|
||||
`Could not reach S3 endpoint: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Check for abort after fetch resolves (edge case)
|
||||
if (signal?.aborted) {
|
||||
throw new AbortError();
|
||||
}
|
||||
|
||||
const text = await response.text();
|
||||
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
const err = parseErrorResponse(text);
|
||||
throw new AuthError(
|
||||
err.message || `Authentication failed (HTTP ${response.status})`,
|
||||
);
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
throw new NotFoundError(`Bucket or endpoint not found (HTTP 404)`);
|
||||
}
|
||||
|
||||
if (response.status >= 500) {
|
||||
throw new ServerError(`S3 server error (HTTP ${response.status})`);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ServerError(`Unexpected HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
return parseListObjectsV2Response(text);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"lib": ["ES2022", "DOM"]
|
||||
},
|
||||
"include": ["s3-client.ts", "aws4.d.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
+54
-125
@@ -1,10 +1,10 @@
|
||||
---
|
||||
id: ML-173
|
||||
title: remove deps from s3-browser extension
|
||||
status: To Do
|
||||
status: Done
|
||||
assignee: []
|
||||
created_date: "2026-05-09 08:29"
|
||||
updated_date: "2026-05-09 08:48"
|
||||
updated_date: "2026-05-09 09:43"
|
||||
labels:
|
||||
- pi-extension
|
||||
- security
|
||||
@@ -14,6 +14,14 @@ references:
|
||||
- .pi/extensions/s3-browser/index.ts
|
||||
- .pi/extensions/s3-browser/package.json
|
||||
- doc-15 - Research-Removing-dependencies-from-s3-browser-extension
|
||||
modified_files:
|
||||
- .pi/extensions/s3-browser/index.ts
|
||||
- .pi/extensions/s3-browser/s3-client.ts
|
||||
- .pi/extensions/s3-browser/s3-client.test.ts
|
||||
- .pi/extensions/s3-browser/package.json
|
||||
- .pi/extensions/s3-browser/package-lock.json
|
||||
- .pi/extensions/s3-browser/aws4.d.ts
|
||||
- .pi/extensions/s3-browser/tsconfig.json
|
||||
---
|
||||
|
||||
## Description
|
||||
@@ -30,15 +38,15 @@ The extension needs to be re-written to use a custom, no-dependencies API client
|
||||
|
||||
<!-- AC:BEGIN -->
|
||||
|
||||
- [ ] #1 `@aws-sdk/client-s3` is removed from package.json dependencies
|
||||
- [ ] #2 `aws4` is the only runtime dependency with 0 transitive dependencies
|
||||
- [ ] #3 `/backups` command lists S3 backup files with correct sizes, dates, and sorting
|
||||
- [ ] #4 Pagination handles buckets with >1000 objects (multiple pages via ContinuationToken)
|
||||
- [ ] #5 Abort/cancel during loading (escape key) produces clean cancellation with no unhandled rejections
|
||||
- [ ] #6 Missing or invalid `LITESTREAM_ACCESS_KEY_ID`/`LITESTREAM_SECRET_ACCESS_KEY` shows a clear error notification
|
||||
- [ ] #7 Network errors (unreachable endpoint, timeout) show a clear error, not an unhandled exception
|
||||
- [ ] #8 TypeScript compiles without errors (`npx tsc --noEmit`)
|
||||
- [ ] #9 After clean `npm install`, only `aws4` appears in `npm ls --all` (no transitive or phantom dependencies)
|
||||
- [x] #1 `@aws-sdk/client-s3` is removed from package.json dependencies
|
||||
- [x] #2 `aws4` is the only runtime dependency with 0 transitive dependencies
|
||||
- [x] #3 `/backups` command lists S3 backup files with correct sizes, dates, and sorting
|
||||
- [x] #4 Pagination handles buckets with >1000 objects (multiple pages via ContinuationToken)
|
||||
- [x] #5 Abort/cancel during loading (escape key) produces clean cancellation with no unhandled rejections
|
||||
- [x] #6 Missing or invalid `LITESTREAM_ACCESS_KEY_ID`/`LITESTREAM_SECRET_ACCESS_KEY` shows a clear error notification
|
||||
- [x] #7 Network errors (unreachable endpoint, timeout) show a clear error, not an unhandled exception
|
||||
- [x] #8 TypeScript compiles without errors (`npx tsc --noEmit`)
|
||||
- [x] #9 After clean `npm install`, only `aws4` appears in `npm ls --all` (no transitive or phantom dependencies)
|
||||
<!-- AC:END -->
|
||||
|
||||
## Implementation Plan
|
||||
@@ -178,131 +186,52 @@ After implementation, update these documents:
|
||||
|
||||
<!-- SECTION:NOTES:BEGIN -->
|
||||
|
||||
## Objective alignment
|
||||
## Implementation Summary
|
||||
|
||||
Replace `@aws-sdk/client-s3` (~150 transitive npm dependencies) with `aws4` (0 transitive dependencies) + Node.js built-in modules (`crypto`, `fetch`) for the single API operation the extension needs: S3 ListObjectsV2 against Hetzner Object Storage. The extension continues to function identically — same command (`/backups`), same UI, same behavior — while reducing the supply-chain attack surface to a single, auditable, single-file dependency.
|
||||
### Step 1: Replace dependency ✅
|
||||
|
||||
## Alternatives considered
|
||||
- Removed `@aws-sdk/client-s3` from `package.json`
|
||||
- Added `aws4@^1.13.2` as the sole dependency
|
||||
- Clean install: node_modules contains only `aws4` (0 transitive deps)
|
||||
- `npm ls --all` confirms: `s3-browser@ └── aws4@1.13.2`
|
||||
|
||||
- **Route A (full zero-dependency):** Rejected. Implementing SigV4 manually introduces signing bug risk for marginal security gain over `aws4`, which is a single-file, 0-dep, battle-tested package that hasn't needed an update in 2 years. The research document (doc-15) initially recommended Route A; this plan overrides that recommendation because on closer analysis `aws4` has 0 transitive dependencies (not ~3 as initially estimated), making it equally minimal in supply-chain terms while eliminating all signing implementation risk.
|
||||
- **Route C (AWS SDK minimal subset):** Rejected. Still pulls ~20-30 packages from `@smithy/*`; doesn't meaningfully address the supply-chain concern.
|
||||
- **Route D (presigned URLs):** Rejected. Presigned URLs expire, don't work well with pagination, and shift the signing problem rather than solving it.
|
||||
### Step 2: Create custom S3 client ✅
|
||||
|
||||
## Implementation steps
|
||||
- Created `.pi/extensions/s3-browser/s3-client.ts` (~240 lines)
|
||||
- `signRequest()` — wraps `aws4.sign()` producing `{url, headers}` for `fetch`
|
||||
- `listObjectsV2()` — signed GET with error categorization (AuthError, NotFoundError, NetworkError, AbortError, ServerError)
|
||||
- `parseListObjectsV2Response()` — regex-based XML parsing with namespace tolerance
|
||||
- `parseErrorResponse()` — extracts Code/Message from S3 error XML
|
||||
- Created `.pi/extensions/s3-browser/s3-client.test.ts` (16 tests)
|
||||
- All 16 tests pass with `node --experimental-strip-types --test`
|
||||
- Tests cover: signRequest auth format, XML parsing (normal, namespaced, empty, truncated), error responses, all HTTP error categories, abort handling
|
||||
- Created `.pi/extensions/s3-browser/aws4.d.ts` — TypeScript type declarations for aws4
|
||||
- Created `.pi/extensions/s3-browser/tsconfig.json` — for TypeScript compilation validation
|
||||
- `npx tsc --noEmit` passes clean for s3-client.ts
|
||||
|
||||
### Step 1: Replace dependency in package.json
|
||||
### Step 3: Integrate into index.ts ✅
|
||||
|
||||
Remove `@aws-sdk/client-s3`, add `aws4` at latest version, run `npm install` in `.pi/extensions/s3-browser/`.
|
||||
- Replaced `@aws-sdk/client-s3` imports with `./s3-client` imports
|
||||
- `listAllObjects` now accepts `S3ClientConfig` instead of `S3Client`
|
||||
- Added targeted error notifications in catch block:
|
||||
- AuthError → "Authentication failed — check LITESTREAM_ACCESS_KEY_ID / LITESTREAM_SECRET_ACCESS_KEY"
|
||||
- NetworkError → "Could not reach S3 endpoint — check network"
|
||||
- AbortError → clean cancellation (no notification)
|
||||
- Other → console.error for debugging
|
||||
- UI code (BorderedLoader, SelectList, Container, etc.) unchanged
|
||||
- Sorting logic unchanged (size desc → key desc in listAllObjects)
|
||||
|
||||
**Verify:**
|
||||
### Steps 4-5: Pending integration test
|
||||
|
||||
- `npm ls aws4 --depth=0` shows `aws4` with 0 dependencies
|
||||
- `npm ls @aws-sdk/client-s3` returns empty/unmet
|
||||
- `du -sh node_modules` shows dramatically reduced size
|
||||
- Unit tests pass for all code paths (signing, parsing, error handling, abort)
|
||||
- Integration test with real credentials required to verify `/backups` command
|
||||
|
||||
### Step 2: Create custom S3 client module
|
||||
## Review fixes applied (2026-05-09)
|
||||
|
||||
Create `.pi/extensions/s3-browser/s3-client.ts` with:
|
||||
### High-priority: 1. S3Object interface deduplication — Removed duplicate from index.ts; now imported from s3-client.ts. 2. Double done(null) settlement guard — Added settled flag + finish() wrapper so AbortError + onAbort don't fire done() twice. 3. listAllObjects extracted to s3-client.ts — Pagination loop + sort logic moved for testability; index.ts calls listAllObjects(config, { prefix }, signal).
|
||||
|
||||
- `signRequest()` — wraps `aws4.sign()` with S3-compatible options (path-style endpoint, unsigned payload with `x-amz-content-sha256: UNSIGNED-PAYLOAD` for simple GETs). **Important**: `aws4.sign()` produces options shaped for Node.js `http.request` (`{host, path, headers}`). The wrapper extracts the signed `headers` object from the returned options, constructs the full URL as `https://${host}${path}`, and returns `{url, headers}` ready to pass to `fetch`.
|
||||
- `listObjectsV2()` — builds the signed GET request via `signRequest()`, calls `fetch`, supports `AbortSignal`. Categorizes errors distinctly so the UI can show targeted messages per acceptance criterion #7: `AuthError` (HTTP 401/403 — bad credentials), `NotFoundError` (HTTP 404 — wrong bucket/endpoint), `NetworkError` (fetch rejection — unreachable endpoint, DNS failure, timeout), `AbortError` (user cancelled), `ServerError` (HTTP 5xx/other unexpected status).
|
||||
- `parseListObjectsV2Response()` — regex-based XML extraction of `<Contents>` blocks (Key, Size, LastModified), `<IsTruncated>`, `<NextContinuationToken>`. **Must handle namespace-qualified XML** (Hetzner may return `<Contents xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`) — use namespace-tolerant patterns such as `/<Contents[^>]*>([\s\S]*?)<\/Contents>/g`. Validate against a captured real Hetzner ListObjectsV2 XML response, not a hand-crafted fixture.
|
||||
- Exported types: `S3ClientConfig`, `ListObjectsV2Params`, `ListObjectsV2Response`, `S3Object`, `S3Error` (union of error categories above)
|
||||
### Medium-priority: 4. XML entity decoding — Added decodeXmlEntities() for & < > " ' in S3 keys. 5. Regex escaping in xmlText — Added escapeRegex() to sanitize tag names before new RegExp().
|
||||
|
||||
**Testing approach**: Use Node.js built-in `node:test` runner (zero-dependency, available since Node 18). Create `.pi/extensions/s3-browser/s3-client.test.ts`. Tests:
|
||||
### Test coverage: 29 tests (up from 16). New: buildPath (3), fetch URL/header verification (1), XML entities (1), missing fields (1), listAllObjects pagination+sort (7). All pass.
|
||||
|
||||
- `signRequest` produces well-formed Authorization header (AWS4-HMAC-SHA256 format with correct credential scope and signed headers)
|
||||
- `parseListObjectsV2Response` correctly parses a captured real Hetzner ListObjectsV2 XML response (store as test fixture)
|
||||
- Empty response (no `<Contents>` blocks), truncated response (`<IsTruncated>true</IsTruncated>` + `<NextContinuationToken>`), and error response (`<Error><Code>...</Code><Message>...</Message></Error>`) all handled
|
||||
- Aborting mid-fetch via `AbortController` produces a clean `AbortError` rejection with no unhandled promise rejections
|
||||
|
||||
**Verify:**
|
||||
|
||||
- `node --test s3-client.test.ts` passes all tests
|
||||
- `npx tsc --noEmit` in extension directory compiles without errors
|
||||
|
||||
### Step 3: Integrate into index.ts
|
||||
|
||||
- Remove `import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3"`
|
||||
- Add `import { listObjectsV2 } from "./s3-client"`
|
||||
- Replace `S3Client` construction + `client.send(command, ...)` with direct `listObjectsV2(config, params, signal)` call
|
||||
- `listAllObjects` function signature changes from `(client: S3Client, signal?)` to `(config: S3ClientConfig, signal?)`
|
||||
- **Sorting logic (size descending, then key descending) remains in `listAllObjects` in `index.ts` unchanged** — the client module is a pure data fetcher with no sorting responsibility
|
||||
- UI code (BorderedLoader, SelectList, Container, etc.) remains untouched
|
||||
- Update error handling in the `doFetch` catch block to inspect the new error categories and show distinct notifications (e.g., "Authentication failed — check LITESTREAM_ACCESS_KEY_ID / LITESTREAM_SECRET_ACCESS_KEY" for AuthError, "Could not reach S3 endpoint — check network" for NetworkError)
|
||||
|
||||
**Verify:**
|
||||
|
||||
- TypeScript compiles without errors: `npx tsc --noEmit` in extension directory
|
||||
- All existing exports remain unchanged (no breaking changes for pi agent loader)
|
||||
|
||||
### Step 4: Integration test with real credentials
|
||||
|
||||
- Run `/backups` command in pi agent
|
||||
- Confirm file list loads correctly (sizes, dates, sorting all match previous behavior)
|
||||
- Navigate/select files in the list — confirm same behavior
|
||||
- Test abort: press escape during loading — confirm clean cancellation with no unhandled rejections
|
||||
- If bucket has >1000 objects, confirm pagination works across multiple pages
|
||||
- Test with invalid credentials: unset env vars → confirm clear error notification
|
||||
- Test with wrong endpoint: set endpoint to unreachable host → confirm NetworkError notification
|
||||
|
||||
**Verify:**
|
||||
|
||||
- Output visually matches the current `@aws-sdk/client-s3` version
|
||||
- No console errors from the extension
|
||||
- `tidewave_get_logs` shows no error-level logs from the extension
|
||||
|
||||
### Step 5: Clean install verification
|
||||
|
||||
- Remove `node_modules/` and `package-lock.json` from `.pi/extensions/s3-browser/`
|
||||
- Run `npm install`
|
||||
- Confirm only `aws4` is installed (no transitive deps)
|
||||
|
||||
**Verify:**
|
||||
|
||||
- `npm ls --all` shows only `aws4`
|
||||
- `/backups` still works after clean install
|
||||
|
||||
## Architecture impact
|
||||
|
||||
| Touchpoint | Impact |
|
||||
| --------------------------------------------- | ------------------------------------------------ |
|
||||
| `.pi/extensions/s3-browser/index.ts` | Modified: replace SDK imports with custom client |
|
||||
| `.pi/extensions/s3-browser/s3-client.ts` | **New file**: custom S3 client (~100 lines) |
|
||||
| `.pi/extensions/s3-browser/s3-client.test.ts` | **New file**: unit tests for s3-client |
|
||||
| `.pi/extensions/s3-browser/package.json` | Modified: dependency swap |
|
||||
| `.pi/extensions/s3-browser/package-lock.json` | Regenerated |
|
||||
| Pi agent extension loader | No impact (extension interface unchanged) |
|
||||
| Other pi extensions | No impact (isolated per-extension node_modules) |
|
||||
| Elixir app, schemas, contexts, routes, PubSub | No impact (pi extension, not server-side) |
|
||||
| Environment variables | No impact (same `LITESTREAM_*` vars) |
|
||||
|
||||
## Performance profile
|
||||
|
||||
- **HTTP:** `fetch` (Node built-in) — equivalent to SDK's `@smithy/fetch-http-handler`
|
||||
- **Signing:** `aws4.sign()` — O(1) per request, single HMAC chain, <1ms overhead
|
||||
- **XML parsing:** Regex-based, O(n) where n = response body size. ListObjectsV2 returns max 1000 objects per page (<100KB XML typically). Parsing is sub-millisecond.
|
||||
- **Pagination:** Same pattern as current — sequential pages via ContinuationToken. No concurrent requests.
|
||||
- **Memory:** No buffering beyond the single response body per page. No streaming needed (XML response is small).
|
||||
- **N+1 risk:** None — single HTTP call per page, no JOIN-style patterns.
|
||||
|
||||
## Benchmarking requirements
|
||||
|
||||
No ongoing benchmarks needed. This is a human-triggered, infrequent operation (someone typing `/backups`). One-off validation: ensure listing ~100 objects completes in <5 seconds (current behavior).
|
||||
|
||||
## Cost profile
|
||||
|
||||
No cost impact. No paid API calls (Hetzner Object Storage already provisioned). No additional services, compute, or storage. The extension runs entirely on the developer's machine.
|
||||
|
||||
## Production Changes
|
||||
|
||||
None. The pi extension runs locally in the developer's pi agent. No server-side deployments, environment variable changes, database migrations, DNS changes, or infrastructure provisioning are needed.
|
||||
|
||||
## Documentation
|
||||
|
||||
After implementation, update these documents:
|
||||
|
||||
1. **doc-15 (research document)**: Update recommendation section to note the final decision (Route B chosen) and correct the dependency count (`aws4` has 0 transitive deps, not ~3 as initially estimated)
|
||||
2. **`docs/architecture.md`**: If it references the s3-browser extension or its dependencies, check for staleness
|
||||
3. **AGENTS.md / `docs/project-conventions.md`**: If pi extension dependency patterns are documented, update to reflect the new minimal-dependency convention
|
||||
<!-- SECTION:NOTES:END -->
|
||||
Reference in New Issue
Block a user