From c5663c4ef90a74dc96a70be21c69357a54aed6db Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Sat, 7 Sep 2024 01:32:08 +0400 Subject: [PATCH] Add prompt finder --- src/Bootstrap.php | 11 +- src/Command/Run.php | 13 ++- src/Module/Finder/Dto/File.php | 17 +++ src/Module/Finder/Dto/Prompt.php | 19 ++++ .../{Finder.php => FilesystemFinder.php} | 7 +- ...inderImpl.php => FilesystemFinderImpl.php} | 21 ++-- src/Module/Finder/Internal/PromptCache.php | 10 ++ .../Finder/Internal/PromptFinderImpl.php | 101 ++++++++++++++++++ src/Module/Finder/PromptFinder.php | 15 +++ 9 files changed, 197 insertions(+), 17 deletions(-) create mode 100644 src/Module/Finder/Dto/File.php create mode 100644 src/Module/Finder/Dto/Prompt.php rename src/Module/Finder/{Finder.php => FilesystemFinder.php} (78%) rename src/Module/Finder/Internal/{FinderImpl.php => FilesystemFinderImpl.php} (73%) create mode 100644 src/Module/Finder/Internal/PromptCache.php create mode 100644 src/Module/Finder/Internal/PromptFinderImpl.php create mode 100644 src/Module/Finder/PromptFinder.php diff --git a/src/Bootstrap.php b/src/Bootstrap.php index a584caf..a977502 100644 --- a/src/Bootstrap.php +++ b/src/Bootstrap.php @@ -4,8 +4,10 @@ namespace LLM\Assistant; -use LLM\Assistant\Module\Finder\Finder; -use LLM\Assistant\Module\Finder\Internal\FinderImpl; +use LLM\Assistant\Module\Finder\FilesystemFinder; +use LLM\Assistant\Module\Finder\Internal\FilesystemFinderImpl; +use LLM\Assistant\Module\Finder\Internal\PromptFinderImpl; +use LLM\Assistant\Module\Finder\PromptFinder; use LLM\Assistant\Service\Cache; use LLM\Assistant\Service\Container; use LLM\Assistant\Service\Internal\Cache\PsrCache; @@ -13,12 +15,10 @@ use LLM\Assistant\Service\Internal\Container\Injection\ConfigLoader; use LLM\Assistant\Service\Internal\Logger\LoggerImpl; use LLM\Assistant\Service\Logger; -use Psr\SimpleCache\CacheInterface; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\StyleInterface; use Symfony\Component\Console\Style\SymfonyStyle; -use Yiisoft\Cache\File\FileCache; /** * Build the container based on the configuration. @@ -48,7 +48,8 @@ public function finish(): Container $c = $this->container; unset($this->container); - $c->bind(Finder::class, FinderImpl::class); + $c->bind(FilesystemFinder::class, FilesystemFinderImpl::class); + $c->bind(PromptFinder::class, PromptFinderImpl::class); $c->bind(Cache::class, PsrCache::class); return $c; diff --git a/src/Command/Run.php b/src/Command/Run.php index 4aef567..6fd587f 100644 --- a/src/Command/Run.php +++ b/src/Command/Run.php @@ -4,7 +4,7 @@ namespace LLM\Assistant\Command; -use LLM\Assistant\Module\Finder\Finder; +use LLM\Assistant\Module\Finder\PromptFinder; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -32,11 +32,14 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->logger->alert('Assistant is running'); - $finder = $this->container->get(Finder::class); + $finder = $this->container->get(PromptFinder::class); - foreach ($finder->files() as $name => $file) { - $this->logger->info($name); - } + #AI test + tr($finder->getNext()); + + // foreach ($finder->files() as $name => $file) { + // $this->logger->info($name); + // } return Command::SUCCESS; } diff --git a/src/Module/Finder/Dto/File.php b/src/Module/Finder/Dto/File.php new file mode 100644 index 0000000..c65305b --- /dev/null +++ b/src/Module/Finder/Dto/File.php @@ -0,0 +1,17 @@ + $prompts + */ + public function __construct( + public readonly \SplFileInfo $file, + public readonly string $content, + public readonly array $prompts, + ) {} +} diff --git a/src/Module/Finder/Dto/Prompt.php b/src/Module/Finder/Dto/Prompt.php new file mode 100644 index 0000000..19d11fb --- /dev/null +++ b/src/Module/Finder/Dto/Prompt.php @@ -0,0 +1,19 @@ + $start + * @param positive-int $length + * @param non-empty-string $prompt + */ + public function __construct( + public readonly int $start, + public readonly int $length, + public readonly string $prompt, + ) {} +} diff --git a/src/Module/Finder/Finder.php b/src/Module/Finder/FilesystemFinder.php similarity index 78% rename from src/Module/Finder/Finder.php rename to src/Module/Finder/FilesystemFinder.php index bf650d8..d3c24d9 100644 --- a/src/Module/Finder/Finder.php +++ b/src/Module/Finder/FilesystemFinder.php @@ -9,7 +9,7 @@ * * @extends \IteratorAggregate */ -interface Finder extends \IteratorAggregate +interface FilesystemFinder extends \IteratorAggregate { /** * @return \Traversable @@ -18,6 +18,11 @@ public function getIterator(): \Traversable; public function after(\DateTimeInterface $date): static; + /** + * Sort by modify time, oldest first + */ + public function oldest(): static; + /** * @return \Traversable */ diff --git a/src/Module/Finder/Internal/FinderImpl.php b/src/Module/Finder/Internal/FilesystemFinderImpl.php similarity index 73% rename from src/Module/Finder/Internal/FinderImpl.php rename to src/Module/Finder/Internal/FilesystemFinderImpl.php index 1508fb2..d58f05d 100644 --- a/src/Module/Finder/Internal/FinderImpl.php +++ b/src/Module/Finder/Internal/FilesystemFinderImpl.php @@ -5,13 +5,17 @@ namespace LLM\Assistant\Module\Finder\Internal; use LLM\Assistant\Config\Source; -use LLM\Assistant\Module\Finder\Finder; +use LLM\Assistant\Module\Finder\FilesystemFinder; -final class FinderImpl implements Finder +final class FilesystemFinderImpl implements FilesystemFinder { + /** + * @param bool|null $sort If true, sort by newest first + */ public function __construct( private readonly Source $config, private ?\DateTimeInterface $after = null, + private ?bool $sort = null, ) {} public function getIterator(): \Traversable @@ -21,7 +25,12 @@ public function getIterator(): \Traversable public function after(\DateTimeInterface $date): static { - return new self($this->config, $date); + return new self($this->config, $date, $this->sort); + } + + public function oldest(): static + { + return new self($this->config, $this->after, false); } public function files(): \Traversable @@ -51,9 +60,9 @@ private function finder(): \Symfony\Component\Finder\Finder $finder->exclude(\array_map($this->directoryToPattern(...), $this->config->excludeDir)); // $finder->exclude($this->config->cacheDir); - if ($this->after !== null) { - $finder->date('> ' . $this->after->format('Y-m-d H:i:s')); - } + $this->after === null or $finder->date('>= ' . $this->after->format('Y-m-d H:i:s')); + // todo direction + $this->sort === null or $finder->sortByModifiedTime(); return $finder; } diff --git a/src/Module/Finder/Internal/PromptCache.php b/src/Module/Finder/Internal/PromptCache.php new file mode 100644 index 0000000..5866396 --- /dev/null +++ b/src/Module/Finder/Internal/PromptCache.php @@ -0,0 +1,10 @@ +getOrUpdateCache(); + + // Get last modified files + $files = $cache->lastScan === null + ? $this->files->oldest()->files() + : $this->files->after($cache->lastScan)->oldest()->files(); + + foreach ($files as $file) { + $result = $this->scanFile($file); + if ($result !== null) { + // Update modified time in cache DTO to be saved later + $mtime = $file->getMTime(); + $mtime === false or $cache->lastScan = new \DateTimeImmutable('@' . $mtime); + + $this->lastCache = $cache; + return $result; + } + } + + return null; + } + + private function getOrUpdateCache(): PromptCache + { + try { + if ($this->lastCache !== null) { + $this->cache->set(self::CACHE_KEY, $this->lastCache); + return $this->lastCache; + } + + return $this->cache->get(self::CACHE_KEY) ?? new PromptCache(); + } catch (\Throwable) { + return new PromptCache(); + } + } + + private function scanFile(\SplFileInfo $file): ?File + { + $content = \file_get_contents($file->getPathname()); + $this->logger->debug( + \sprintf( + 'Scanning MT: %s Path: %s', + (new \DateTimeImmutable('@' . (int) $file->getMTime()))->format('Y-m-d H:i:s'), + $file->getPathname(), + ), + ); + + // Find all inline prompts started with "#AI" from a new line + $matches = []; + $result = \preg_match_all( + '/^[ \\t]*#AI[ \\t]+(.+)$/m', + $content, + $matches, + \PREG_OFFSET_CAPTURE, + ); + + if ($result === 0) { + return null; + } + + $prompts = []; + /** + * @var array{ + * list}>, + * list + * } $matches + */ + foreach ($matches[0] as $key => $match) { + $prompts[] = new Prompt($match[1], \strlen($match[0]), $matches[1][$key][0]); + } + + return new File($file, $content, $prompts); + } +} diff --git a/src/Module/Finder/PromptFinder.php b/src/Module/Finder/PromptFinder.php new file mode 100644 index 0000000..356d7b8 --- /dev/null +++ b/src/Module/Finder/PromptFinder.php @@ -0,0 +1,15 @@ +