Compare commits

...

3 Commits

Author SHA1 Message Date
Zecheng Zhang 61d66e1580 Keep S3 request handler runtime-only 2026-07-24 05:31:48 -07:00
Zecheng Zhang 9a2b6a50bd Remove TypeScript changeset 2026-07-24 05:14:24 -07:00
Zecheng Zhang c3b99d3133 Fix TypeScript S3 proxy and blob decoding 2026-07-24 05:11:55 -07:00
26 changed files with 248 additions and 49 deletions
+7 -3
View File
@@ -16,10 +16,14 @@ import { Accessor } from './base.ts'
import type { Resource } from '../resource/base.ts'
import type { S3Config } from '../resource/s3/config.ts'
export class S3Accessor extends Accessor {
readonly config: S3Config
export interface S3RuntimeConfig extends S3Config {
requestHandler?: unknown
}
constructor(config: S3Config) {
export class S3Accessor extends Accessor {
readonly config: S3RuntimeConfig
constructor(config: S3RuntimeConfig) {
super()
this.config = config
}
@@ -0,0 +1,59 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { afterEach, describe, expect, it, vi } from 'vitest'
import { LanceDBAccessor } from '../../accessor/lancedb.ts'
import { resolveLanceDBConfig } from '../../resource/lancedb/config.ts'
import { PathSpec } from '../../types.ts'
import type { LanceDriver } from './_driver.ts'
import { read } from './read.ts'
const BLOB_PATH = new PathSpec({ resourcePath: '1.bin', virtual: '/1.bin', directory: '/1.bin' })
function makeAccessor(blob: unknown): LanceDBAccessor {
const driver = {
rowRecord: vi.fn().mockResolvedValue({ id: '1', blob }),
} as unknown as LanceDriver
const config = resolveLanceDBConfig({
uri: '/tmp/db',
table: 'items',
idColumn: 'id',
blobColumn: 'blob',
blobExt: 'bin',
})
return new LanceDBAccessor(driver, config)
}
describe('lancedb core read', () => {
afterEach(() => {
vi.unstubAllGlobals()
})
it('decodes base64 without the Node Buffer global', async () => {
vi.stubGlobal('Buffer', undefined)
const bytes = await read(makeAccessor('AAEC/w=='), BLOB_PATH)
expect([...bytes]).toEqual([0, 1, 2, 255])
})
it('returns Uint8Array blobs unchanged', async () => {
const blob = new Uint8Array([3, 2, 1])
expect(await read(makeAccessor(blob), BLOB_PATH)).toBe(blob)
})
it('rejects values that are neither bytes nor base64 strings', async () => {
await expect(read(makeAccessor(42), BLOB_PATH)).rejects.toThrow(
'blob column is not bytes or base64 string',
)
})
})
@@ -16,6 +16,7 @@ import type { LanceDBAccessor } from '../../accessor/lancedb.ts'
import type { IndexCacheStore } from '../../cache/index/store.ts'
import type { LanceRow } from './_driver.ts'
import { PathSpec } from '../../types.ts'
import { decodeBase64 } from '../../utils/base64.ts'
import { renderCard } from './render.ts'
import { type LanceDBScope, ScopeLevel, detectScope } from './scope.ts'
@@ -35,7 +36,7 @@ async function resolveRow(accessor: LanceDBAccessor, scope: LanceDBScope): Promi
function blobBytes(value: unknown): Uint8Array {
if (value instanceof Uint8Array) return value
if (typeof value === 'string') return Uint8Array.from(Buffer.from(value, 'base64'))
if (typeof value === 'string') return decodeBase64(value)
throw new Error('blob column is not bytes or base64 string')
}
@@ -0,0 +1,25 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it, vi } from 'vitest'
import { createS3Client } from './_client.ts'
describe('S3 client', () => {
it('forwards an injected request handler', async () => {
const requestHandler = { handle: vi.fn() }
const client = await createS3Client({ bucket: 'b', requestHandler })
expect(client.config.requestHandler).toBe(requestHandler)
client.destroy()
})
})
@@ -15,6 +15,7 @@
import { mountPrefixOf } from '../../utils/key_prefix.ts'
import type { S3Client } from '@aws-sdk/client-s3'
import type { PathSpec } from '../../types.ts'
import type { S3RuntimeConfig } from '../../accessor/s3.ts'
import { loadOptionalPeer } from '../../utils/optional_peer.ts'
import * as kp from '../../utils/key_prefix.ts'
import type { S3Config } from '../../resource/s3/config.ts'
@@ -44,7 +45,7 @@ export interface S3SendClient {
}
export async function withClient<T>(
config: S3Config,
config: S3RuntimeConfig,
fn: (client: S3SendClient) => Promise<T>,
): Promise<T> {
const client = (await createS3Client(config)) as unknown as S3SendClient
@@ -93,7 +94,7 @@ export async function loadS3Module(config?: S3Config): Promise<S3Module> {
return cachedModule
}
export async function createS3Client(config: S3Config): Promise<S3Client> {
export async function createS3Client(config: S3RuntimeConfig): Promise<S3Client> {
if (config.presignedUrlProvider !== undefined) {
const { createBrowserS3Client } = await import('./_client_browser.ts')
return createBrowserS3Client(config) as unknown as S3Client
@@ -110,7 +111,9 @@ export async function createS3Client(config: S3Config): Promise<S3Client> {
...(config.sessionToken !== undefined ? { sessionToken: config.sessionToken } : {}),
}
}
if (config.timeoutMs !== undefined) {
if (config.requestHandler !== undefined) {
options.requestHandler = config.requestHandler
} else if (config.timeoutMs !== undefined) {
options.requestHandler = {
connectionTimeout: config.timeoutMs,
requestTimeout: config.timeoutMs,
+3
View File
@@ -49,11 +49,14 @@
},
"dependencies": {
"@napi-rs/lzma": "^1.4.5",
"@smithy/node-http-handler": "^4.7.3",
"@struktoai/mirage-core": "workspace:*",
"compressjs": "^1.0.3",
"fast-xml-parser": "^5.9.0",
"fast-xml-validator": "^1.4.0",
"fs-monkey": "^1.1.0",
"http-proxy-agent": "^7.0.2",
"https-proxy-agent": "^7.0.6",
"tree-sitter-bash": "^0.25.1",
"web-tree-sitter": "^0.26.8"
},
@@ -24,6 +24,7 @@ export interface AliyunConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface AliyunConfigRedacted {
@@ -35,6 +36,7 @@ export interface AliyunConfigRedacted {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const AliyunConfigSchema = z.object({
@@ -46,6 +48,7 @@ const AliyunConfigSchema = z.object({
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function resolvedAliyunEndpoint(config: AliyunConfig): string {
@@ -64,6 +67,7 @@ export function aliyunToS3Config(config: AliyunConfig): S3Config {
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -87,6 +91,5 @@ export function normalizeAliyunConfig(input: Record<string, unknown>): AliyunCon
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as AliyunConfig
}
@@ -24,6 +24,7 @@ export interface BackblazeConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface BackblazeConfigRedacted {
@@ -35,6 +36,7 @@ export interface BackblazeConfigRedacted {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const BackblazeConfigSchema = z.object({
@@ -46,6 +48,7 @@ const BackblazeConfigSchema = z.object({
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function resolvedBackblazeEndpoint(config: BackblazeConfig): string {
@@ -64,6 +67,7 @@ export function backblazeToS3Config(config: BackblazeConfig): S3Config {
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -87,6 +91,5 @@ export function normalizeBackblazeConfig(input: Record<string, unknown>): Backbl
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as BackblazeConfig
}
@@ -24,6 +24,7 @@ export interface CephConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface CephConfigRedacted {
@@ -35,6 +36,7 @@ export interface CephConfigRedacted {
forcePathStyle: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const CephConfigSchema = z.object({
@@ -46,6 +48,7 @@ const CephConfigSchema = z.object({
forcePathStyle: z.boolean(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function cephToS3Config(config: CephConfig): S3Config {
@@ -58,6 +61,7 @@ export function cephToS3Config(config: CephConfig): S3Config {
forcePathStyle: config.forcePathStyle ?? true,
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -82,6 +86,5 @@ export function normalizeCephConfig(input: Record<string, unknown>): CephConfig
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as CephConfig
}
@@ -24,6 +24,7 @@ export interface DigitalOceanConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface DigitalOceanConfigRedacted {
@@ -35,6 +36,7 @@ export interface DigitalOceanConfigRedacted {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const DigitalOceanConfigSchema = z.object({
@@ -46,6 +48,7 @@ const DigitalOceanConfigSchema = z.object({
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function resolvedDigitalOceanEndpoint(config: DigitalOceanConfig): string {
@@ -64,6 +67,7 @@ export function digitalOceanToS3Config(config: DigitalOceanConfig): S3Config {
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -87,6 +91,5 @@ export function normalizeDigitalOceanConfig(input: Record<string, unknown>): Dig
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as DigitalOceanConfig
}
@@ -26,6 +26,7 @@ export interface GCSConfig {
timeoutMs?: number
forcePathStyle?: boolean
keyPrefix?: string
proxy?: string
}
export interface GCSConfigRedacted {
@@ -37,6 +38,7 @@ export interface GCSConfigRedacted {
timeoutMs?: number
forcePathStyle?: boolean
keyPrefix?: string
proxy?: string
}
const GCSConfigSchema = z.object({
@@ -48,6 +50,7 @@ const GCSConfigSchema = z.object({
timeoutMs: z.number().optional(),
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
proxy: secretStr().optional(),
})
export function gcsToS3Config(config: GCSConfig): S3Config {
@@ -60,6 +63,7 @@ export function gcsToS3Config(config: GCSConfig): S3Config {
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -84,6 +88,5 @@ export function normalizeGcsConfig(input: Record<string, unknown>): GCSConfig {
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as GCSConfig
}
@@ -24,6 +24,7 @@ export interface MinIOConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface MinIOConfigRedacted {
@@ -35,6 +36,7 @@ export interface MinIOConfigRedacted {
forcePathStyle: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const MinIOConfigSchema = z.object({
@@ -46,6 +48,7 @@ const MinIOConfigSchema = z.object({
forcePathStyle: z.boolean(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function minioToS3Config(config: MinIOConfig): S3Config {
@@ -58,6 +61,7 @@ export function minioToS3Config(config: MinIOConfig): S3Config {
forcePathStyle: config.forcePathStyle ?? true,
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -82,6 +86,5 @@ export function normalizeMinIOConfig(input: Record<string, unknown>): MinIOConfi
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as MinIOConfig
}
@@ -24,6 +24,7 @@ export interface OCIConfig {
endpoint?: string
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface OCIConfigRedacted {
@@ -35,6 +36,7 @@ export interface OCIConfigRedacted {
endpoint: string
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const OCIConfigSchema = z.object({
@@ -46,6 +48,7 @@ const OCIConfigSchema = z.object({
endpoint: z.string(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
function resolvedOciEndpoint(config: OCIConfig): string {
@@ -63,6 +66,7 @@ export function ociToS3Config(config: OCIConfig): S3Config {
forcePathStyle: true,
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -85,6 +89,5 @@ export function normalizeOciConfig(input: Record<string, unknown>): OCIConfig {
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as OCIConfig
}
@@ -24,6 +24,7 @@ export interface QingStorConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface QingStorConfigRedacted {
@@ -35,6 +36,7 @@ export interface QingStorConfigRedacted {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const QingStorConfigSchema = z.object({
@@ -46,6 +48,7 @@ const QingStorConfigSchema = z.object({
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function resolvedQingStorEndpoint(config: QingStorConfig): string {
@@ -64,6 +67,7 @@ export function qingStorToS3Config(config: QingStorConfig): S3Config {
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -87,6 +91,5 @@ export function normalizeQingStorConfig(input: Record<string, unknown>): QingSto
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as QingStorConfig
}
@@ -26,6 +26,7 @@ export interface R2Config {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface R2ConfigRedacted {
@@ -39,6 +40,7 @@ export interface R2ConfigRedacted {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const R2ConfigSchema = z.object({
@@ -52,6 +54,7 @@ const R2ConfigSchema = z.object({
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
function resolvedR2Endpoint(config: R2Config): string {
@@ -73,6 +76,7 @@ export function r2ToS3Config(config: R2Config): S3Config {
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -99,6 +103,5 @@ export function normalizeR2Config(input: Record<string, unknown>): R2Config {
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as R2Config
}
@@ -97,7 +97,7 @@ describe('node resource registry', () => {
endpoint_url: 'https://example.com',
path_style: true,
timeout: 30,
proxy: 'http://discarded',
proxy: 'http://proxy.example',
})) as unknown as { config: Record<string, unknown> }
expect(config).toMatchObject({
bucket: 'b',
@@ -109,8 +109,8 @@ describe('node resource registry', () => {
endpoint: 'https://example.com',
forcePathStyle: true,
timeoutMs: 30_000,
proxy: 'http://proxy.example',
})
expect(config).not.toHaveProperty('proxy')
})
it('S3: accepts already-camelCase keys (TS-idiomatic)', async () => {
@@ -164,6 +164,7 @@ describe('node resource registry', () => {
accessKeyId: 'A',
endpoint: 'https://x',
timeoutMs: 5_000,
proxy: 'p',
})
})
})
@@ -18,19 +18,23 @@ import {
S3ConfigSchema as S3CoreConfigSchema,
type S3Config as S3CoreConfig,
type S3ConfigRedacted as S3CoreConfigRedacted,
secretStr,
z,
} from '@struktoai/mirage-core'
export interface S3Config extends S3CoreConfig {
profile?: string
proxy?: string
}
export interface S3ConfigRedacted extends S3CoreConfigRedacted {
profile?: string
proxy?: string
}
const S3ConfigSchema = S3CoreConfigSchema.extend({
profile: z.string().optional(),
proxy: secretStr().optional(),
})
export function redactConfig(config: S3Config): S3ConfigRedacted {
@@ -51,7 +55,7 @@ export function redactConfig(config: S3Config): S3ConfigRedacted {
* endpoint_url ↔ endpoint
* path_style ↔ forcePathStyle
* timeout (sec, int) ↔ timeoutMs (ms, number — converted ×1000)
* proxy ↔ (dropped — not yet supported in TS)
* proxy ↔ proxy
*/
export function normalizeS3Config(input: Record<string, unknown>): S3Config {
return normalizeFields(input, {
@@ -68,6 +72,5 @@ export function normalizeS3Config(input: Record<string, unknown>): S3Config {
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as S3Config
}
@@ -13,6 +13,7 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { PathSpec, mountKey } from '@struktoai/mirage-core'
import { NodeHttpHandler } from '@smithy/node-http-handler'
import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'
import { S3Resource } from './s3.ts'
import type { S3Config } from './config.ts'
@@ -59,6 +60,19 @@ describe('S3Resource credential redaction', () => {
expect(serialized).not.toContain('TOKEN-OBVIOUS-LEAK')
expect(serialized).toContain('<REDACTED>')
})
it('configures a Node request handler and redacts proxy credentials', async () => {
const res = new S3Resource({
bucket: 'b',
proxy: 'http://proxy-user:proxy-secret@localhost:8080',
timeoutMs: 1234,
})
expect(res.accessor.config.requestHandler).toBeInstanceOf(NodeHttpHandler)
const serialized = JSON.stringify(await res.getState())
expect(serialized).not.toContain('proxy-user')
expect(serialized).not.toContain('proxy-secret')
expect(serialized).toContain('<REDACTED>')
})
})
describe('S3Resource (mocked integration)', () => {
+17 -1
View File
@@ -49,10 +49,21 @@ import {
unlink as unlinkCore,
write as writeCore,
} from '@struktoai/mirage-core'
import { NodeHttpHandler } from '@smithy/node-http-handler'
import { HttpProxyAgent } from 'http-proxy-agent'
import { HttpsProxyAgent } from 'https-proxy-agent'
import { redactConfig, type S3Config, type S3ConfigRedacted } from './config.ts'
const globCore = makeResolveGlob(readdirCore, S3_SCOPE_ERROR)
function createProxyRequestHandler(proxy: string, timeoutMs: number | undefined): NodeHttpHandler {
return new NodeHttpHandler({
httpAgent: new HttpProxyAgent(proxy),
httpsAgent: new HttpsProxyAgent(proxy),
...(timeoutMs !== undefined ? { connectionTimeout: timeoutMs, requestTimeout: timeoutMs } : {}),
})
}
export interface S3ResourceState {
type: string
config: S3ConfigRedacted
@@ -97,7 +108,12 @@ export class S3Resource extends BaseResource implements Resource {
delete cfg.keyPrefix
}
this.config = cfg
this.accessor = new S3Accessor(this.config)
this.accessor = new S3Accessor({
...cfg,
...(cfg.proxy !== undefined
? { requestHandler: createProxyRequestHandler(cfg.proxy, cfg.timeoutMs) }
: {}),
})
}
open(): Promise<void> {
@@ -13,8 +13,9 @@
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { describe, expect, it } from 'vitest'
import { ResourceName, type S3Config as S3CoreConfig } from '@struktoai/mirage-core'
import { ResourceName } from '@struktoai/mirage-core'
import { S3Resource } from './s3/s3.ts'
import type { S3Config } from './s3/config.ts'
import {
aliyunToS3Config,
normalizeAliyunConfig,
@@ -162,13 +163,18 @@ describe('region-derived S3 aliases', () => {
})
it(`${c.name}: toS3Config maps fields`, () => {
const s3 = c.toS3({ ...c.make(c.region), timeoutMs: 5000 } as never)
const s3 = c.toS3({
...c.make(c.region),
timeoutMs: 5000,
proxy: 'http://localhost:8080',
} as never)
expect(s3.bucket).toBe('b')
expect(s3.region).toBe(c.region)
expect(s3.endpoint).toBe(c.expectedEndpoint)
expect(s3.accessKeyId).toBe('AKIA-LEAK')
expect(s3.secretAccessKey).toBe('SECRET-LEAK')
expect(s3.timeoutMs).toBe(5000)
expect(s3.proxy).toBe('http://localhost:8080')
expect(s3.forcePathStyle).toBeUndefined()
})
@@ -190,7 +196,7 @@ describe('region-derived S3 aliases', () => {
expect(norm.forcePathStyle).toBe(true)
expect(norm.keyPrefix).toBe('team/reports')
expect(norm.timeoutMs).toBe(30000)
expect(norm).not.toHaveProperty('proxy')
expect(norm.proxy).toBe('http://localhost:8080')
expect(norm).not.toHaveProperty('access_key_id')
})
@@ -204,11 +210,14 @@ describe('region-derived S3 aliases', () => {
})
it(`${c.name}: getState redacts creds`, async () => {
const state = await c.build(c.make(c.region) as never).getState()
const state = await c
.build({ ...c.make(c.region), proxy: 'http://user:secret@localhost:8080' } as never)
.getState()
expect(state.type).toBe(c.kind)
const blob = JSON.stringify(state)
expect(blob.includes('AKIA-LEAK')).toBe(false)
expect(blob.includes('SECRET-LEAK')).toBe(false)
expect(blob.includes('user:secret')).toBe(false)
expect(blob.includes('<REDACTED>')).toBe(true)
})
}
@@ -247,7 +256,7 @@ describe('wasabi endpoint defaults', () => {
expect(norm.forcePathStyle).toBe(true)
expect(norm.keyPrefix).toBe('team/reports')
expect(norm.timeoutMs).toBe(30000)
expect(norm).not.toHaveProperty('proxy')
expect(norm.proxy).toBe('p')
})
it('resource remaps kind and redacts state', async () => {
@@ -264,7 +273,7 @@ const ENDPOINT_CASES = [
{
name: 'minio',
kind: ResourceName.MINIO,
toS3: minioToS3Config as (config: never) => S3CoreConfig,
toS3: minioToS3Config as (config: never) => S3Config,
normalize: normalizeMinIOConfig as (input: Record<string, unknown>) => unknown,
make: (): MinIOConfig => ({ ...CREDS, endpoint: 'http://localhost:9000' }),
build: (config: never) => new MinIOResource(config),
@@ -272,7 +281,7 @@ const ENDPOINT_CASES = [
{
name: 'ceph',
kind: ResourceName.CEPH,
toS3: cephToS3Config as (config: never) => S3CoreConfig,
toS3: cephToS3Config as (config: never) => S3Config,
normalize: normalizeCephConfig as (input: Record<string, unknown>) => unknown,
make: (): CephConfig => ({ ...CREDS, endpoint: 'http://localhost:9000' }),
build: (config: never) => new CephResource(config),
@@ -280,7 +289,7 @@ const ENDPOINT_CASES = [
{
name: 'seaweedfs',
kind: ResourceName.SEAWEEDFS,
toS3: seaweedfsToS3Config as (config: never) => S3CoreConfig,
toS3: seaweedfsToS3Config as (config: never) => S3Config,
normalize: normalizeSeaweedFSConfig as (input: Record<string, unknown>) => unknown,
make: (): SeaweedFSConfig => ({ ...CREDS, endpoint: 'http://localhost:9000' }),
build: (config: never) => new SeaweedFSResource(config),
@@ -317,7 +326,7 @@ describe('endpoint-required S3 aliases (minio/ceph/seaweedfs)', () => {
expect(norm.forcePathStyle).toBe(false)
expect(norm.keyPrefix).toBe('team/reports')
expect(norm.timeoutMs).toBe(30000)
expect(norm).not.toHaveProperty('proxy')
expect(norm.proxy).toBe('p')
})
it(`${c.name}: resource remaps kind and redacts state`, async () => {
@@ -338,72 +347,72 @@ const PREFIX_CASES = [
{
name: 'aliyun',
config: { ...CREDS, region: 'us-east-1', forcePathStyle: true } as AliyunConfig,
toS3: aliyunToS3Config as (config: never) => S3CoreConfig,
toS3: aliyunToS3Config as (config: never) => S3Config,
},
{
name: 'backblaze',
config: { ...CREDS, region: 'us-east-1', forcePathStyle: true } as BackblazeConfig,
toS3: backblazeToS3Config as (config: never) => S3CoreConfig,
toS3: backblazeToS3Config as (config: never) => S3Config,
},
{
name: 'ceph',
config: { ...CREDS, endpoint: 'http://localhost:9000' } as CephConfig,
toS3: cephToS3Config as (config: never) => S3CoreConfig,
toS3: cephToS3Config as (config: never) => S3Config,
},
{
name: 'digitalocean',
config: { ...CREDS, region: 'us-east-1', forcePathStyle: true } as DigitalOceanConfig,
toS3: digitalOceanToS3Config as (config: never) => S3CoreConfig,
toS3: digitalOceanToS3Config as (config: never) => S3Config,
},
{
name: 'gcs',
config: { ...CREDS, forcePathStyle: true } as GCSConfig,
toS3: gcsToS3Config as (config: never) => S3CoreConfig,
toS3: gcsToS3Config as (config: never) => S3Config,
},
{
name: 'minio',
config: { ...CREDS, endpoint: 'http://localhost:9000' } as MinIOConfig,
toS3: minioToS3Config as (config: never) => S3CoreConfig,
toS3: minioToS3Config as (config: never) => S3Config,
},
{
name: 'oci',
config: { ...CREDS, namespace: 'ns', region: 'us-east-1' } as OCIConfig,
toS3: ociToS3Config as (config: never) => S3CoreConfig,
toS3: ociToS3Config as (config: never) => S3Config,
},
{
name: 'qingstor',
config: { ...CREDS, region: 'us-east-1', forcePathStyle: true } as QingStorConfig,
toS3: qingStorToS3Config as (config: never) => S3CoreConfig,
toS3: qingStorToS3Config as (config: never) => S3Config,
},
{
name: 'r2',
config: { ...CREDS, accountId: 'account', forcePathStyle: true } as R2Config,
toS3: r2ToS3Config as (config: never) => S3CoreConfig,
toS3: r2ToS3Config as (config: never) => S3Config,
},
{
name: 'scaleway',
config: { ...CREDS, region: 'us-east-1', forcePathStyle: true } as ScalewayConfig,
toS3: scalewayToS3Config as (config: never) => S3CoreConfig,
toS3: scalewayToS3Config as (config: never) => S3Config,
},
{
name: 'seaweedfs',
config: { ...CREDS, endpoint: 'http://localhost:9000' } as SeaweedFSConfig,
toS3: seaweedfsToS3Config as (config: never) => S3CoreConfig,
toS3: seaweedfsToS3Config as (config: never) => S3Config,
},
{
name: 'supabase',
config: { ...CREDS, projectRef: 'project', region: 'us-east-1' } as SupabaseConfig,
toS3: supabaseToS3Config as (config: never) => S3CoreConfig,
toS3: supabaseToS3Config as (config: never) => S3Config,
},
{
name: 'tencent',
config: { ...CREDS, region: 'us-east-1', forcePathStyle: true } as TencentConfig,
toS3: tencentToS3Config as (config: never) => S3CoreConfig,
toS3: tencentToS3Config as (config: never) => S3Config,
},
{
name: 'wasabi',
config: { ...CREDS, forcePathStyle: true } as WasabiConfig,
toS3: wasabiToS3Config as (config: never) => S3CoreConfig,
toS3: wasabiToS3Config as (config: never) => S3Config,
},
] as const
@@ -415,4 +424,14 @@ describe('S3 alias subfolder mounts', () => {
expect(s3.forcePathStyle).toBe(true)
})
}
for (const c of PREFIX_CASES) {
it(`${c.name}: forwards proxy`, () => {
const s3 = c.toS3({
...c.config,
proxy: 'http://localhost:8080',
} as never)
expect(s3.proxy).toBe('http://localhost:8080')
})
}
})
@@ -24,6 +24,7 @@ export interface ScalewayConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface ScalewayConfigRedacted {
@@ -35,6 +36,7 @@ export interface ScalewayConfigRedacted {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const ScalewayConfigSchema = z.object({
@@ -46,6 +48,7 @@ const ScalewayConfigSchema = z.object({
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function resolvedScalewayEndpoint(config: ScalewayConfig): string {
@@ -64,6 +67,7 @@ export function scalewayToS3Config(config: ScalewayConfig): S3Config {
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -87,6 +91,5 @@ export function normalizeScalewayConfig(input: Record<string, unknown>): Scalewa
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as ScalewayConfig
}
@@ -24,6 +24,7 @@ export interface SeaweedFSConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface SeaweedFSConfigRedacted {
@@ -35,6 +36,7 @@ export interface SeaweedFSConfigRedacted {
forcePathStyle: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const SeaweedFSConfigSchema = z.object({
@@ -46,6 +48,7 @@ const SeaweedFSConfigSchema = z.object({
forcePathStyle: z.boolean(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function seaweedfsToS3Config(config: SeaweedFSConfig): S3Config {
@@ -58,6 +61,7 @@ export function seaweedfsToS3Config(config: SeaweedFSConfig): S3Config {
forcePathStyle: config.forcePathStyle ?? true,
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -82,6 +86,5 @@ export function normalizeSeaweedFSConfig(input: Record<string, unknown>): Seawee
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as SeaweedFSConfig
}
@@ -25,6 +25,7 @@ export interface SupabaseConfig {
sessionToken?: string
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface SupabaseConfigRedacted {
@@ -37,6 +38,7 @@ export interface SupabaseConfigRedacted {
sessionToken?: string
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const SupabaseConfigSchema = z.object({
@@ -49,6 +51,7 @@ const SupabaseConfigSchema = z.object({
sessionToken: secretStr().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function resolvedSupabaseEndpoint(config: SupabaseConfig): string {
@@ -70,6 +73,7 @@ export function supabaseToS3Config(config: SupabaseConfig): S3Config {
...(config.sessionToken !== undefined ? { sessionToken: config.sessionToken } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -94,6 +98,5 @@ export function normalizeSupabaseConfig(input: Record<string, unknown>): Supabas
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as SupabaseConfig
}
@@ -24,6 +24,7 @@ export interface TencentConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface TencentConfigRedacted {
@@ -35,6 +36,7 @@ export interface TencentConfigRedacted {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const TencentConfigSchema = z.object({
@@ -46,6 +48,7 @@ const TencentConfigSchema = z.object({
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function resolvedTencentEndpoint(config: TencentConfig): string {
@@ -64,6 +67,7 @@ export function tencentToS3Config(config: TencentConfig): S3Config {
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -87,6 +91,5 @@ export function normalizeTencentConfig(input: Record<string, unknown>): TencentC
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as TencentConfig
}
@@ -24,6 +24,7 @@ export interface WasabiConfig {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
export interface WasabiConfigRedacted {
@@ -35,6 +36,7 @@ export interface WasabiConfigRedacted {
forcePathStyle?: boolean
keyPrefix?: string
timeoutMs?: number
proxy?: string
}
const WasabiConfigSchema = z.object({
@@ -46,6 +48,7 @@ const WasabiConfigSchema = z.object({
forcePathStyle: z.boolean().optional(),
keyPrefix: z.string().optional(),
timeoutMs: z.number().optional(),
proxy: secretStr().optional(),
})
export function resolvedWasabiEndpoint(config: WasabiConfig): string {
@@ -64,6 +67,7 @@ export function wasabiToS3Config(config: WasabiConfig): S3Config {
...(config.forcePathStyle !== undefined ? { forcePathStyle: config.forcePathStyle } : {}),
...(config.keyPrefix !== undefined ? { keyPrefix: config.keyPrefix } : {}),
...(config.timeoutMs !== undefined ? { timeoutMs: config.timeoutMs } : {}),
...(config.proxy !== undefined ? { proxy: config.proxy } : {}),
}
}
@@ -88,6 +92,5 @@ export function normalizeWasabiConfig(input: Record<string, unknown>): WasabiCon
transform: {
timeout: (v: unknown) => (typeof v === 'number' ? v * 1000 : v),
},
drop: ['proxy'],
}) as unknown as WasabiConfig
}
+9
View File
@@ -491,6 +491,9 @@ importers:
'@napi-rs/lzma':
specifier: ^1.4.5
version: 1.4.5(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)
'@smithy/node-http-handler':
specifier: ^4.7.3
version: 4.7.3
'@struktoai/mirage-core':
specifier: workspace:*
version: link:../core
@@ -506,6 +509,12 @@ importers:
fs-monkey:
specifier: ^1.1.0
version: 1.1.0
http-proxy-agent:
specifier: ^7.0.2
version: 7.0.2
https-proxy-agent:
specifier: ^7.0.6
version: 7.0.6
tree-sitter-bash:
specifier: ^0.25.1
version: 0.25.1