TokenTextSplitter

TokenTextSplitter = object

Defined in: packages/cloud/src/client/types.gen.ts:5899

Implementation of splitting text that looks at word tokens.

Properties

include_metadata?

optional include_metadata: boolean

Defined in: packages/cloud/src/client/types.gen.ts:5903

Whether or not to consider metadata when splitting.

---

include_prev_next_rel?

optional include_prev_next_rel: boolean

Defined in: packages/cloud/src/client/types.gen.ts:5907

Include prev/next node relationships.

---

callback_manager?

optional callback_manager: unknown

Defined in: packages/cloud/src/client/types.gen.ts:5908

---

id_func?

optional id_func: string | null

Defined in: packages/cloud/src/client/types.gen.ts:5912

Function to generate node IDs.

---

chunk_size?

optional chunk_size: number

Defined in: packages/cloud/src/client/types.gen.ts:5916

The token chunk size for each chunk.

---

chunk_overlap?

optional chunk_overlap: number

Defined in: packages/cloud/src/client/types.gen.ts:5920

The token overlap of each chunk when splitting.

---

separator?

optional separator: string

Defined in: packages/cloud/src/client/types.gen.ts:5924

Default separator for splitting into words

---

backup_separators?

optional backup_separators: unknown[]

Defined in: packages/cloud/src/client/types.gen.ts:5928

Additional separators for splitting.

---

class_name?

optional class_name: string

Defined in: packages/cloud/src/client/types.gen.ts:5929