SentenceSplitter

Defined in: packages/core/src/node-parser/sentence-splitter.ts:24

Parse text with a preference for complete sentences.

Extends

• MetadataAwareTextSplitter

SentenceSplitter<Options>(nodes, options?): TextNode<Metadata>[]

Defined in: packages/core/src/node-parser/sentence-splitter.ts:24

Parse text with a preference for complete sentences.

Type Parameters

Options

Options extends Record<string, unknown>

Parameters

nodes

BaseNode<Metadata>[]

options?

Options

Returns

TextNode<Metadata>[]

Constructors

Constructor

new SentenceSplitter(params?): SentenceSplitter

Defined in: packages/core/src/node-parser/sentence-splitter.ts:51

Parameters

params?

object & SplitterParams

Returns

SentenceSplitter

Overrides

MetadataAwareTextSplitter.constructor

Properties

includeMetadata

includeMetadata: boolean = true

Defined in: packages/core/src/node-parser/base.ts:17

Inherited from

MetadataAwareTextSplitter.includeMetadata

---

includePrevNextRel

includePrevNextRel: boolean = true

Defined in: packages/core/src/node-parser/base.ts:18

Inherited from

MetadataAwareTextSplitter.includePrevNextRel

---

chunkSize

chunkSize: number = 1024

Defined in: packages/core/src/node-parser/sentence-splitter.ts:28

The token chunk size for each chunk.

---

chunkOverlap

chunkOverlap: number = 200

Defined in: packages/core/src/node-parser/sentence-splitter.ts:32

The token overlap of each chunk when splitting.

---

separator

separator: string = " "

Defined in: packages/core/src/node-parser/sentence-splitter.ts:36

Default separator for splitting into words

---

paragraphSeparator

paragraphSeparator: string = "\n\n\n"

Defined in: packages/core/src/node-parser/sentence-splitter.ts:40

Separator between paragraphs.

---

secondaryChunkingRegex

secondaryChunkingRegex: string = "[^,.;。？！]+[,.;。？！]?"

Defined in: packages/core/src/node-parser/sentence-splitter.ts:44

Backup regex for splitting into sentences.

---

id

id: string

Defined in: packages/core/src/schema/type.ts:22

Inherited from

MetadataAwareTextSplitter.id

Methods

postProcessParsedNodes()

protected postProcessParsedNodes(nodes, parentDocMap): TextNode<Metadata>[]

Defined in: packages/core/src/node-parser/base.ts:27

Parameters

nodes

TextNode<Metadata>[]

parentDocMap

Map<string, TextNode<Metadata>>

Returns

TextNode<Metadata>[]

Inherited from

MetadataAwareTextSplitter.postProcessParsedNodes

---

getNodesFromDocuments()

getNodesFromDocuments(documents): TextNode<Metadata>[]

Defined in: packages/core/src/node-parser/base.ts:84

Parameters

documents

TextNode<Metadata>[]

Returns

TextNode<Metadata>[]

Inherited from

MetadataAwareTextSplitter.getNodesFromDocuments

---

splitTexts()

splitTexts(texts): string[]

Defined in: packages/core/src/node-parser/base.ts:126

Parameters

texts

string[]

Returns

string[]

Inherited from

MetadataAwareTextSplitter.splitTexts

---

splitTextsMetadataAware()

splitTextsMetadataAware(texts, metadata): string[]

Defined in: packages/core/src/node-parser/base.ts:142

Parameters

texts

string[]

metadata

string[]

Returns

string[]

Inherited from

MetadataAwareTextSplitter.splitTextsMetadataAware

---

getMetadataString()

protected getMetadataString(node): string

Defined in: packages/core/src/node-parser/base.ts:151

Parameters

node

TextNode

Returns

string

Inherited from

MetadataAwareTextSplitter.getMetadataString

---

parseNodes()

protected parseNodes(nodes): TextNode<Metadata>[]

Defined in: packages/core/src/node-parser/base.ts:161

Parameters

nodes

TextNode<Metadata>[]

Returns

TextNode<Metadata>[]

Inherited from

MetadataAwareTextSplitter.parseNodes

---

splitTextMetadataAware()

splitTextMetadataAware(text, metadata): string[]

Defined in: packages/core/src/node-parser/sentence-splitter.ts:72

Parameters

text

string

metadata

string

Returns

string[]

Overrides

MetadataAwareTextSplitter.splitTextMetadataAware

---

splitText()

splitText(text): string[]

Defined in: packages/core/src/node-parser/sentence-splitter.ts:87

Parameters

text

string

Returns

string[]

Overrides

MetadataAwareTextSplitter.splitText

---

_splitText()

_splitText(text, chunkSize): string[]

Defined in: packages/core/src/node-parser/sentence-splitter.ts:91

Parameters

text

string

chunkSize

number

Returns

string[]

---

tokenSize()

tokenSize(text): number

Defined in: packages/core/src/node-parser/sentence-splitter.ts:225

Parameters

text

string

Returns

number