File tree 1 file changed +8
-1
lines changed
1 file changed +8
-1
lines changed Original file line number Diff line number Diff line change @@ -150,7 +150,7 @@ def __chunk_node(
150
150
f"Traversing at node { node .text .decode ()} at position { node .byte_range } "
151
151
)
152
152
current_chunk : str = ""
153
-
153
+ prev_node = None
154
154
current_start = None
155
155
156
156
for child in node .children :
@@ -184,10 +184,17 @@ def __chunk_node(
184
184
current_start = Point (
185
185
row = child .start_point .row + 1 , column = child .start_point .column
186
186
)
187
+ prev_node = child
187
188
188
189
elif len (current_chunk ) + child_length <= self .config .chunk_size :
189
190
# Add to current chunk
191
+ if prev_node :
192
+ if prev_node .end_point .row != child .start_point .row :
193
+ current_chunk += "\n "
194
+ else :
195
+ current_chunk += " "
190
196
current_chunk += child_bytes .decode ()
197
+ prev_node = child
191
198
192
199
else :
193
200
# Yield current chunk and start new one
You can’t perform that action at this time.
0 commit comments