@terascope/file-asset-apis / s3/s3-reader-api / S3TerasliceAPI

Class: S3TerasliceAPI

Defined in: s3/s3-reader-api.ts:13

Extends

• S3Fetcher

Constructors

Constructor

new S3TerasliceAPI(client, config, logger): S3TerasliceAPI

Defined in: s3/s3-reader-api.ts:17

Parameters

Parameter	Type
client	S3Client
config	ReaderConfig
logger	Logger

Returns

S3TerasliceAPI

Overrides

S3Fetcher.constructor

Properties

Property	Modifier	Type	Inherited from	Defined in
bucket	readonly	string	S3Fetcher.bucket	s3/s3-fetcher.ts:9
client	protected	S3Client	S3Fetcher.client	s3/s3-fetcher.ts:8
compressor	public	Compressor	S3Fetcher.compressor	base/chunked-file-reader.ts:74
filePerSlice	protected	boolean	S3Fetcher.filePerSlice	base/chunked-file-reader.ts:83
logger	public	Logger	S3Fetcher.logger	base/chunked-file-reader.ts:73
segmentFileConfig	readonly	SliceConfig	-	s3/s3-reader-api.ts:14
slicerConfig	readonly	FileSliceConfig	-	s3/s3-reader-api.ts:15

Accessors

format

Get Signature

get format(): Format

Defined in: base/chunked-file-reader.ts:127

Returns

Format

Inherited from

S3Fetcher.format

---

lineDelimiter

Get Signature

get lineDelimiter(): string

Defined in: base/chunked-file-reader.ts:123

Returns

string

Inherited from

S3Fetcher.lineDelimiter

Methods

canReadFile()

canReadFile(filePath): boolean

Defined in: s3/s3-reader-api.ts:40

Determines if a file name or file path can be processed, it will return false if the name of path contains a segment that starts with "."

Parameters

Parameter	Type
filePath	string

Returns

boolean

---

csv()

protected csv(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:233

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

S3Fetcher.csv

---

fetch()

protected fetch(slice): Promise<string>

Defined in: s3/s3-fetcher.ts:27

low level api that fetches the unprocessed contents of the file, please use the "read" method for correct file and data parsing

Parameters

Parameter	Type
slice	FileSlice

Returns

Promise<string>

Example

const slice = { offset: 0, length: 1000, path: 'some/file.txt', total: 1000 };
  const results = await s3Reader.fetch(slice);
  results === 'the unprocessed contents of the file here'

Inherited from

S3Fetcher.fetch

---

getMargin()

protected getMargin(slice, delimiter): Promise<string>

Defined in: base/chunked-file-reader.ts:200

Parameters

Parameter	Type
slice	FileSlice
delimiter	string

Returns

Promise<string>

Inherited from

S3Fetcher.getMargin

---

json()

protected json(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:294

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

S3Fetcher.json

---

ldjson()

protected ldjson(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:320

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

S3Fetcher.ldjson

---

makeSlicer()

makeSlicer(): Promise<() => Promise<null | FileSlice[]>>

Defined in: s3/s3-reader-api.ts:101

Generates a function that will resolve one or more slices each time it is called. These slices will can be used to "fetch" chunks of data. Returns null when complete

Returns

Promise<() => Promise<null | FileSlice[]>>

Example

const config = {
     size: 1000,
     file_per_slice: false,
     line_delimiter: '\n',
     size: 300,
     format: "ldjson"
     path: 'some/dir'
  }
  const s3Reader = new S3Reader(config);
  const slicer = await s3Reader.newSlicer();

  const results = await slicer();
  results === [
     { offset: 0, length: 1000, path: 'some/dir/file.txt', total: 1000 }
  ]

---

raw()

protected raw(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:224

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

S3Fetcher.raw

---

read()

read(slice): Promise<DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>[]>

Defined in: base/chunked-file-reader.ts:334

Parameters

Parameter	Type
slice	FileSlice

Returns

Promise<DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>[]>

Inherited from

S3Fetcher.read

---

segmentFile()

segmentFile(file): FileSlice[]

Defined in: s3/s3-reader-api.ts:73

Used to slice up a file based on the configuration provided

The returned results can be used directly with any "read" method of a reader API

Parameters

Parameter	Type
file	{ path: string; size: number; }
file.path	string
file.size	number

Returns

FileSlice[]

Example

const slice = { path: 'some/path', size: 1000 };
    const config = {
       file_per_slice: false,
       line_delimiter: '\n',
       size: 300,
       format: Format.ldjson
    };

    const results = s3Reader.segmentFile(slice, config);

    results === [
       {
           offset: 0, length: 300, path: 'some/path', total: 1000
       },
           offset: 299, length: 301, path: 'some/path', total: 1000
       },
           offset: 599, length: 301, path: 'some/path', total: 1000
       },
       {
           offset: 899, length: 101, path: 'some/path', total: 1000
       }
    ]

---

tsv()

protected tsv(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:288

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

S3Fetcher.tsv