@terascope/file-asset-apis / file/file-reader-api / FileTerasliceAPI

Class: FileTerasliceAPI

Defined in: file/file-reader-api.ts:13

Extends

• FileFetcher

Constructors

Constructor

new FileTerasliceAPI(config, logger): FileTerasliceAPI

Defined in: file/file-reader-api.ts:17

Parameters

Parameter	Type
config	ReaderConfig
logger	Logger

Returns

FileTerasliceAPI

Overrides

FileFetcher.constructor

Properties

Property	Modifier	Type	Default value	Inherited from	Defined in
client	public	__module	fse	FileFetcher.client	file/file-fetcher.ts:7
compressor	public	Compressor	undefined	FileFetcher.compressor	base/chunked-file-reader.ts:74
filePerSlice	protected	boolean	undefined	FileFetcher.filePerSlice	base/chunked-file-reader.ts:83
logger	public	Logger	undefined	FileFetcher.logger	base/chunked-file-reader.ts:73
segmentFileConfig	readonly	SliceConfig	undefined	-	file/file-reader-api.ts:14
size	readonly	number	undefined	FileFetcher.size	file/file-fetcher.ts:8
slicerConfig	readonly	FileSliceConfig	undefined	-	file/file-reader-api.ts:15

Accessors

format

Get Signature

get format(): Format

Defined in: base/chunked-file-reader.ts:127

Returns

Format

Inherited from

FileFetcher.format

---

lineDelimiter

Get Signature

get lineDelimiter(): string

Defined in: base/chunked-file-reader.ts:123

Returns

string

Inherited from

FileFetcher.lineDelimiter

Methods

canReadFile()

canReadFile(fileName): boolean

Defined in: file/file-reader-api.ts:40

Determines if a file name or file path can be processed, it will return false if the name of path contains a segment that starts with "."

Parameters

Parameter	Type
fileName	string

Returns

boolean

---

csv()

protected csv(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:233

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

FileFetcher.csv

---

fetch()

protected fetch(slice): Promise<string>

Defined in: file/file-fetcher.ts:29

low level api that fetches the unprocessed contents of the file, please use the "read" method for correct file and data parsing

Parameters

Parameter	Type
slice	FileSlice

Returns

Promise<string>

Example

const slice = {
         offset: 0,
         length: 1000,
         path: 'some/file.txt',
         total: 1000
     };
     const results = await fileReader.fetch(slice);
     results === 'the unprocessed contents of the file here'

Inherited from

FileFetcher.fetch

---

getMargin()

protected getMargin(slice, delimiter): Promise<string>

Defined in: base/chunked-file-reader.ts:200

Parameters

Parameter	Type
slice	FileSlice
delimiter	string

Returns

Promise<string>

Inherited from

FileFetcher.getMargin

---

json()

protected json(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:294

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

FileFetcher.json

---

ldjson()

protected ldjson(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:320

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

FileFetcher.ldjson

---

makeSlicer()

makeSlicer(): Promise<() => Promise<null | FileSlice[]>>

Defined in: file/file-reader-api.ts:137

Generates a function that will resolve one or more slices each time it is called. These slices will can be used to "fetch" chunks of data. Returns null when complete

Returns

Promise<() => Promise<null | FileSlice[]>>

Example

const config = {
     size: 1000,
     file_per_slice: false,
     line_delimiter: '\n',
     size: 300,
     format: "ldjson"
     path: 'some/dir'
  }
  const fileReader = new FileReader(config);
  const slicer = await fileReader.newSlicer();

  const results = await slicer();
  results === [
     {
         offset: 0,
         length: 1000,
         path: 'some/dir/file.txt',
         total: 1000
     }
  ]

---

raw()

protected raw(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:224

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

FileFetcher.raw

---

read()

read(slice): Promise<DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>[]>

Defined in: base/chunked-file-reader.ts:334

Parameters

Parameter	Type
slice	FileSlice

Returns

Promise<DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>[]>

Inherited from

FileFetcher.read

---

segmentFile()

segmentFile(file): FileSlice[]

Defined in: file/file-reader-api.ts:103

Create file segments from input, these will can be used to fetch specific chunks of a file

Parameters

Parameter	Type
file	{ path: string; size: number; }
file.path	string
file.size	number

Returns

FileSlice[]

Example

const config = {
     size: 1000,
     file_per_slice: false,
     line_delimiter: '\n',
     size: 300,
     format: "ldjson"
  }
  const fileReader = new FileReader(config);
  const results = fileReader.segmentFile({
     path: 'some/file.txt',
     size: 2000
  });
  results === [
     {
         offset: 0,
         length: 1000,
         path: 'some/file.txt',
         total: 1000
     },
     {
         offset: 1001,
         length: 1000,
         path: 'some/file.txt',
         total: 1000
     },
  ]

---

tsv()

protected tsv(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:288

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

FileFetcher.tsv

---

validatePath()

validatePath(path): void

Defined in: file/file-reader-api.ts:54

Determines if a directory can be processed, will throw if it is a symlink or if the directory is empty

Parameters

Parameter	Type
path	string

Returns

void

Example

fileReader.validatePath('some/emptyDir') => Error
  fileReader.validatePath('some/symlinkedDir')  => Error
  fileReader.validatePath('some/dir') => void