@terascope/file-asset-apis / hdfs/hdfs-fetcher / HDFSReader

Class: HDFSReader

Defined in: hdfs/hdfs-fetcher.ts:5

Extends

• ChunkedFileReader

Constructors

Constructor

new HDFSReader(client, config, logger): HDFSReader

Defined in: hdfs/hdfs-fetcher.ts:8

Parameters

Parameter	Type
client	AnyObject
config	HDFSReaderConfig
logger	Logger

Returns

HDFSReader

Overrides

ChunkedFileReader.constructor

Properties

Property	Modifier	Type	Inherited from	Defined in
client	public	AnyObject	-	hdfs/hdfs-fetcher.ts:6
compressor	public	Compressor	ChunkedFileReader.compressor	base/chunked-file-reader.ts:74
filePerSlice	protected	boolean	ChunkedFileReader.filePerSlice	base/chunked-file-reader.ts:83
logger	public	Logger	ChunkedFileReader.logger	base/chunked-file-reader.ts:73

Accessors

format

Get Signature

get format(): Format

Defined in: base/chunked-file-reader.ts:127

Returns

Format

Inherited from

ChunkedFileReader.format

---

lineDelimiter

Get Signature

get lineDelimiter(): string

Defined in: base/chunked-file-reader.ts:123

Returns

string

Inherited from

ChunkedFileReader.lineDelimiter

Methods

csv()

protected csv(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:233

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

ChunkedFileReader.csv

---

fetch()

protected fetch(slice): Promise<string>

Defined in: hdfs/hdfs-fetcher.ts:21

low level api that fetches the unprocessed contents of the file from HDFS, please use the "read" method for correct file and data parsing

Parameters

Parameter	Type
slice	FileSlice

Returns

Promise<string>

Example

const slice = { offset: 0, length: 1000, path: 'some/file.txt', total: 1000 };
  const results = await hdfsReader.fetch(slice);
  results === 'the unprocessed contents of the file here'

Overrides

ChunkedFileReader.fetch

---

getMargin()

protected getMargin(slice, delimiter): Promise<string>

Defined in: base/chunked-file-reader.ts:200

Parameters

Parameter	Type
slice	FileSlice
delimiter	string

Returns

Promise<string>

Inherited from

ChunkedFileReader.getMargin

---

json()

protected json(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:294

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

ChunkedFileReader.json

---

ldjson()

protected ldjson(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:320

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

ChunkedFileReader.ldjson

---

raw()

protected raw(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:224

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

ChunkedFileReader.raw

---

read()

read(slice): Promise<DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>[]>

Defined in: base/chunked-file-reader.ts:334

Parameters

Parameter	Type
slice	FileSlice

Returns

Promise<DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>[]>

Inherited from

ChunkedFileReader.read

---

tsv()

protected tsv(incomingData, slice): Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Defined in: base/chunked-file-reader.ts:288

Parameters

Parameter	Type
incomingData	string
slice	FileSlice

Returns

Promise<(null | DataEntity<Record<string, any>, _DataEntityMetadata<Record<string, any>>>)[]>

Inherited from

ChunkedFileReader.tsv