@terascope/file-asset-apis / s3/s3-fetcher / S3Fetcher

Class: S3Fetcher

Defined in: s3/s3-fetcher.ts:7

Extends

• ChunkedFileReader

Extended by

• S3TerasliceAPI

Constructors

Constructor

new S3Fetcher(client, config, logger): S3Fetcher

Defined in: s3/s3-fetcher.ts:11

Parameters

Parameter	Type
client	S3Client
config	Omit<ReaderAPIConfig, "size">
logger	Logger

Returns

S3Fetcher

Overrides

ChunkedFileReader.constructor

Properties

Property	Modifier	Type	Inherited from	Defined in
bucket	readonly	string	-	s3/s3-fetcher.ts:9
client	protected	S3Client	-	s3/s3-fetcher.ts:8
compressor	public	Compressor	ChunkedFileReader.compressor	base/chunked-file-reader.ts:73
filePerSlice	protected	boolean	ChunkedFileReader.filePerSlice	base/chunked-file-reader.ts:82
logger	public	Logger	ChunkedFileReader.logger	base/chunked-file-reader.ts:72

Accessors

format

Get Signature

get format(): Format

Defined in: base/chunked-file-reader.ts:126

Returns

Format

Inherited from

ChunkedFileReader.format

---

lineDelimiter

Get Signature

get lineDelimiter(): string

Defined in: base/chunked-file-reader.ts:122

Returns

string

Inherited from

ChunkedFileReader.lineDelimiter

Methods

csv()

protected csv(incomingData, slice): Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Defined in: base/chunked-file-reader.ts:232

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Inherited from

ChunkedFileReader.csv

---

fetch()

protected fetch(slice): Promise<string>

Defined in: s3/s3-fetcher.ts:27

low level api that fetches the unprocessed contents of the file, please use the "read" method for correct file and data parsing

Parameters

Parameter	Type
slice	FileSlice

Returns

Promise<string>

Example

const slice = { offset: 0, length: 1000, path: 'some/file.txt', total: 1000 };
  const results = await s3Reader.fetch(slice);
  results === 'the unprocessed contents of the file here'

Overrides

ChunkedFileReader.fetch

---

getMargin()

protected getMargin(slice, delimiter): Promise<string>

Defined in: base/chunked-file-reader.ts:199

Parameters

Parameter	Type
slice	FileSlice
delimiter	string

Returns

Promise<string>

Inherited from

ChunkedFileReader.getMargin

---

json()

protected json(incomingData, slice): Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Defined in: base/chunked-file-reader.ts:293

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Inherited from

ChunkedFileReader.json

---

ldjson()

protected ldjson(incomingData, slice): Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Defined in: base/chunked-file-reader.ts:319

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Inherited from

ChunkedFileReader.ldjson

---

raw()

protected raw(incomingData, slice): Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Defined in: base/chunked-file-reader.ts:223

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Inherited from

ChunkedFileReader.raw

---

read()

read(slice): Promise<DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>[]>

Defined in: base/chunked-file-reader.ts:333

Parameters

Parameter	Type
slice	FileSlice

Returns

Promise<DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>[]>

Inherited from

ChunkedFileReader.read

---

tsv()

protected tsv(incomingData, slice): Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Defined in: base/chunked-file-reader.ts:287

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>> | null)[]>

Inherited from

ChunkedFileReader.tsv