sentence splitting implementation

.gitignore

@@ -0,0 +1 @@
+.DS_Store

Embeddings.php

@@ -336,11 +336,16 @@ protected function splitIntoChunks($text)
         while ($sentence = array_shift($sentences)) {
             $slen = count($tiktok->encode($sentence));
             if ($slen > $this->getChunkSize()) {
-                // sentence is too long, we need to split it further
-                if ($this->logger instanceof CLI) $this->logger->warning(
-                    'Sentence too long, splitting not implemented yet'
-                );
-                continue;
+                // Sentence is too long, split into smaller parts
+                if ($this->logger instanceof CLI) {
+                    $this->logger->warning(
+                        'Sentence too long, splitting it into smaller parts'
+                    );
+                }
+
+                // Push split sentences to the front of the queue
+                array_unshift($sentences, ...$this->splitLongSentence($sentence, $tiktok));
+                continue; // Restart loop with the newly inserted sentences
             }
 
             if ($chunklen + $slen < $this->getChunkSize()) {
@@ -360,11 +365,51 @@ protected function splitIntoChunks($text)
                 $chunklen = count($tiktok->encode($chunk));
             }
         }
-        $chunks[] = $chunk;
-
+
+        // Add the last chunk if not empty
+        if (trim($chunk) !== '') $chunks[] = trim($chunk);
+
         return $chunks;
     }
 
+    protected function splitLongSentence($sentence, $tiktok)
+    {
+        $words = preg_split('/(\s+)/', $sentence, -1, PREG_SPLIT_DELIM_CAPTURE);
+        $subSentences = [];
+        $currentSubSentence = '';
+        $currentSubSentenceLen = 0;
+        $chunkSize = $this->getChunkSize();
+
+        foreach ($words as $word) {
+            $wordLen = count($tiktok->encode($word));
+
+            if ($wordLen > $chunkSize) {
+                // If a single word is too long, split it into smaller chunks
+                $wordChunks = str_split($word, intval($chunkSize / 2)); // Split into smaller parts
+                foreach ($wordChunks as $chunk) {
+                    $subSentences[] = trim($chunk);
+                }
+            } elseif ($currentSubSentenceLen + $wordLen < $chunkSize) {
+                // Add to current sub-sentence
+                $currentSubSentence .= $word;
+                $currentSubSentenceLen += $wordLen;
+            } else {
+                // Add current sub-sentence to result
+                $subSentences[] = trim($currentSubSentence);
+                // Start new sub-sentence
+                $currentSubSentence = $word;
+                $currentSubSentenceLen = $wordLen;
+            }
+        }
+
+        // Add last sub-sentence to result
+        if ($currentSubSentence !== '') {
+            $subSentences[] = trim($currentSubSentence);
+        }
+
+        return $subSentences;
+    }
+
     /**
      * Add a sentence to the queue of remembered sentences
      *