TokenTextSplitter

Defined in: packages/core/src/node-parser/token-text-splitter.ts:17

Extends

• MetadataAwareTextSplitter

TokenTextSplitter<Options>(nodes, options?): TextNode<Metadata>[]

Defined in: packages/core/src/node-parser/token-text-splitter.ts:17

Type Parameters

Options

Options extends Record<string, unknown>

Parameters

nodes

BaseNode<Metadata>[]

options?

Options

Returns

TextNode<Metadata>[]

Constructors

Constructor

new TokenTextSplitter(params?): TokenTextSplitter

Defined in: packages/core/src/node-parser/token-text-splitter.ts:25

Parameters

params?

SplitterParams & Partial<{ chunkSize: number; chunkOverlap: number; separator: string; backupSeparators: string[]; }>

Returns

TokenTextSplitter

Overrides

MetadataAwareTextSplitter.constructor

Properties

includeMetadata

includeMetadata: boolean = true

Defined in: packages/core/src/node-parser/base.ts:17

Inherited from

MetadataAwareTextSplitter.includeMetadata

---

includePrevNextRel

includePrevNextRel: boolean = true

Defined in: packages/core/src/node-parser/base.ts:18

Inherited from

MetadataAwareTextSplitter.includePrevNextRel

---

chunkSize

chunkSize: number = DEFAULT_CHUNK_SIZE

Defined in: packages/core/src/node-parser/token-text-splitter.ts:18

---

chunkOverlap

chunkOverlap: number = DEFAULT_CHUNK_OVERLAP

Defined in: packages/core/src/node-parser/token-text-splitter.ts:19

---

separator

separator: string = " "

Defined in: packages/core/src/node-parser/token-text-splitter.ts:20

---

backupSeparators

backupSeparators: string[]

Defined in: packages/core/src/node-parser/token-text-splitter.ts:21

---

id

id: string

Defined in: packages/core/src/schema/type.ts:22

Inherited from

MetadataAwareTextSplitter.id

Methods

postProcessParsedNodes()

protected postProcessParsedNodes(nodes, parentDocMap): TextNode<Metadata>[]

Defined in: packages/core/src/node-parser/base.ts:27

Parameters

nodes

TextNode<Metadata>[]

parentDocMap

Map<string, TextNode<Metadata>>

Returns

TextNode<Metadata>[]

Inherited from

MetadataAwareTextSplitter.postProcessParsedNodes

---

getNodesFromDocuments()

getNodesFromDocuments(documents): TextNode<Metadata>[]

Defined in: packages/core/src/node-parser/base.ts:84

Parameters

documents

TextNode<Metadata>[]

Returns

TextNode<Metadata>[]

Inherited from

MetadataAwareTextSplitter.getNodesFromDocuments

---

splitTexts()

splitTexts(texts): string[]

Defined in: packages/core/src/node-parser/base.ts:126

Parameters

texts

string[]

Returns

string[]

Inherited from

MetadataAwareTextSplitter.splitTexts

---

splitTextsMetadataAware()

splitTextsMetadataAware(texts, metadata): string[]

Defined in: packages/core/src/node-parser/base.ts:142

Parameters

texts

string[]

metadata

string[]

Returns

string[]

Inherited from

MetadataAwareTextSplitter.splitTextsMetadataAware

---

getMetadataString()

protected getMetadataString(node): string

Defined in: packages/core/src/node-parser/base.ts:151

Parameters

node

TextNode

Returns

string

Inherited from

MetadataAwareTextSplitter.getMetadataString

---

parseNodes()

protected parseNodes(nodes): TextNode<Metadata>[]

Defined in: packages/core/src/node-parser/base.ts:161

Parameters

nodes

TextNode<Metadata>[]

Returns

TextNode<Metadata>[]

Inherited from

MetadataAwareTextSplitter.parseNodes

---

splitTextMetadataAware()

splitTextMetadataAware(text, metadata): string[]

Defined in: packages/core/src/node-parser/token-text-splitter.ts:57

Split text into chunks, reserving space required for metadata string.

Parameters

text

string

The text to split.

metadata

string

The metadata string.

Returns

string[]

An array of text chunks.

Overrides

MetadataAwareTextSplitter.splitTextMetadataAware

---

splitText()

splitText(text): string[]

Defined in: packages/core/src/node-parser/token-text-splitter.ts:82

Split text into chunks.

Parameters

text

string

The text to split.

Returns

string[]

An array of text chunks.

Overrides

MetadataAwareTextSplitter.splitText