From 3a8dc340bbc42536bb7a527a689f117186b2b920 Mon Sep 17 00:00:00 2001 From: VBoss Date: Tue, 22 Oct 2019 15:55:27 +0200 Subject: [PATCH 01/87] Lower utils --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 2957ffe2..6a65b97b 100644 --- a/composer.json +++ b/composer.json @@ -28,7 +28,7 @@ "ext-json": "*", "nette/di": "^2.4.10", "nette/security": "^2.4.3", - "nette/utils": "^2.5.3", + "nette/utils": "^2.4.10", "spameri/elastic-query": "^0.5.0", "elasticsearch/elasticsearch": "^6.0", "kdyby/console": "^2.7.1", From f65acb307f67ed6ab23ed31d45cbdb3dc5267982 Mon Sep 17 00:00:00 2001 From: VBoss Date: Tue, 22 Oct 2019 16:02:11 +0200 Subject: [PATCH 02/87] Dependency fun --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 6a65b97b..e3b166b2 100644 --- a/composer.json +++ b/composer.json @@ -27,7 +27,7 @@ "php": ">=7.1", "ext-json": "*", "nette/di": "^2.4.10", - "nette/security": "^2.4.3", + "nette/security": "^2.3.0", "nette/utils": "^2.4.10", "spameri/elastic-query": "^0.5.0", "elasticsearch/elasticsearch": "^6.0", From b09bfe317993ef955bc1efad9067c48e701f5e92 Mon Sep 17 00:00:00 2001 From: VBoss Date: Tue, 22 Oct 2019 16:08:44 +0200 Subject: [PATCH 03/87] Dependency fun --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index e3b166b2..b3dc58ca 100644 --- a/composer.json +++ b/composer.json @@ -34,7 +34,7 @@ "kdyby/console": "^2.7.1", "kdyby/datetime-provider": "v1.0.0", "kdyby/monolog": "^1.3.2", - "tracy/tracy": "^2.6.0" + "tracy/tracy": "^2.4.18" }, "require-dev": { "spameri/coding-standard": "dev-master", From 6cfa999411612ecc46013f464dc551a524d5fe8e Mon Sep 17 00:00:00 2001 From: VBoss Date: Tue, 8 Oct 2019 14:04:49 +0200 Subject: [PATCH 04/87] Data to Elastic --- composer.json | 5 +- src/Commands/InitializeSettings.php | 54 ++++++ src/Commands/Migrate.php | 38 ++++ src/Config/Elastic.neon | 27 ++- ....php => SpameriElasticSearchExtension.php} | 4 +- src/Diagnostics/PanelLogger.php | 2 + src/Entity/AbstractImport.php | 59 +++++++ src/Entity/Import/ArrayValue.php | 40 +++++ src/Entity/Import/BoolValue.php | 41 +++++ src/Entity/Import/DateValue.php | 48 ++++++ src/Entity/Import/EmptyValue.php | 34 ++++ src/Entity/Import/FloatValue.php | 41 +++++ src/Entity/Import/IntegerValue.php | 41 +++++ src/Entity/Import/NoValue.php | 19 ++ src/Entity/Import/StringValue.php | 41 +++++ .../Import/ValidationObjectInterface.php | 12 ++ .../Import/ValidationPropertyInterface.php | 12 ++ src/Exception/SettingsNotLocated.php | 16 ++ src/Import/AfterImport/NullAfterImport.php | 16 ++ src/Import/AfterImportInterface.php | 13 ++ src/Import/DataImportInterface.php | 12 ++ src/Import/DataProviderInterface.php | 17 ++ src/Import/Exception/AlreadyLocked.php | 8 + src/Import/Exception/Error.php | 11 ++ src/Import/Exception/Fatal.php | 11 ++ src/Import/Exception/ImportException.php | 9 + src/Import/Exception/Omit.php | 11 ++ src/Import/Lock/FileLock.php | 66 +++++++ src/Import/Lock/NullLock.php | 31 ++++ src/Import/LockInterface.php | 25 +++ src/Import/LoggerHandlerInterface.php | 40 +++++ src/Import/PrepareImportDataInterface.php | 12 ++ src/Import/Response/SimpleResponse.php | 46 +++++ src/Import/ResponseInterface.php | 19 ++ src/Import/Run.php | 163 ++++++++++++++++++ src/Import/Run/LoggerHandler.php | 63 +++++++ src/Import/Run/NullLoggerHandler.php | 49 ++++++ src/Import/Run/Options.php | 27 +++ src/Import/RunHandler/ConsoleHandler.php | 30 ++++ src/Import/RunHandler/NullHandler.php | 27 +++ src/Import/RunHandlerInterface.php | 21 +++ src/Import/SimpleRun.php | 23 +++ src/Migration/AbstractMigration.php | 42 +++++ src/Migration/MigrationFileInterface.php | 15 ++ src/Migration/MigrationRunner.php | 26 +++ src/Model/BaseService.php | 19 +- src/Model/EntitySettingsLocator.php | 35 ++++ src/Model/Indices/Delete.php | 44 +++++ src/Model/Insert/PrepareEntityArray.php | 4 + src/Settings/IndexConfigInterface.php | 10 ++ tests/SpameriTests/Data/Config/Video.neon | 3 +- .../Data/Entity/Video/Identification.php | 2 + .../2019/09/24/BasicData/Initialize.php | 28 +++ .../2019/09/24/Structure/Initialize.php | 65 +++++++ 54 files changed, 1557 insertions(+), 20 deletions(-) create mode 100644 src/Commands/InitializeSettings.php create mode 100644 src/Commands/Migrate.php rename src/DI/{ElasticSearchExtension.php => SpameriElasticSearchExtension.php} (96%) create mode 100644 src/Entity/AbstractImport.php create mode 100644 src/Entity/Import/ArrayValue.php create mode 100644 src/Entity/Import/BoolValue.php create mode 100644 src/Entity/Import/DateValue.php create mode 100644 src/Entity/Import/EmptyValue.php create mode 100644 src/Entity/Import/FloatValue.php create mode 100644 src/Entity/Import/IntegerValue.php create mode 100644 src/Entity/Import/NoValue.php create mode 100644 src/Entity/Import/StringValue.php create mode 100644 src/Entity/Import/ValidationObjectInterface.php create mode 100644 src/Entity/Import/ValidationPropertyInterface.php create mode 100644 src/Exception/SettingsNotLocated.php create mode 100644 src/Import/AfterImport/NullAfterImport.php create mode 100644 src/Import/AfterImportInterface.php create mode 100644 src/Import/DataImportInterface.php create mode 100644 src/Import/DataProviderInterface.php create mode 100644 src/Import/Exception/AlreadyLocked.php create mode 100644 src/Import/Exception/Error.php create mode 100644 src/Import/Exception/Fatal.php create mode 100644 src/Import/Exception/ImportException.php create mode 100644 src/Import/Exception/Omit.php create mode 100644 src/Import/Lock/FileLock.php create mode 100644 src/Import/Lock/NullLock.php create mode 100644 src/Import/LockInterface.php create mode 100644 src/Import/LoggerHandlerInterface.php create mode 100644 src/Import/PrepareImportDataInterface.php create mode 100644 src/Import/Response/SimpleResponse.php create mode 100644 src/Import/ResponseInterface.php create mode 100644 src/Import/Run.php create mode 100644 src/Import/Run/LoggerHandler.php create mode 100644 src/Import/Run/NullLoggerHandler.php create mode 100644 src/Import/Run/Options.php create mode 100644 src/Import/RunHandler/ConsoleHandler.php create mode 100644 src/Import/RunHandler/NullHandler.php create mode 100644 src/Import/RunHandlerInterface.php create mode 100644 src/Import/SimpleRun.php create mode 100644 src/Migration/AbstractMigration.php create mode 100644 src/Migration/MigrationFileInterface.php create mode 100644 src/Migration/MigrationRunner.php create mode 100644 src/Model/EntitySettingsLocator.php create mode 100644 src/Model/Indices/Delete.php create mode 100644 src/Settings/IndexConfigInterface.php create mode 100644 tests/SpameriTests/Elastic/Migration/migrations/2019/09/24/BasicData/Initialize.php create mode 100644 tests/SpameriTests/Elastic/Migration/migrations/2019/09/24/Structure/Initialize.php diff --git a/composer.json b/composer.json index 2957ffe2..48698b65 100644 --- a/composer.json +++ b/composer.json @@ -54,6 +54,9 @@ "autoload-dev": { "psr-4": { "SpameriTests\\": "tests/SpameriTests" - } + }, + "classmap": [ + "tests/SpameriTests/Elastic/Migration/migrations" + ] } } diff --git a/src/Commands/InitializeSettings.php b/src/Commands/InitializeSettings.php new file mode 100644 index 00000000..0984a2ff --- /dev/null +++ b/src/Commands/InitializeSettings.php @@ -0,0 +1,54 @@ +indexConfig = $indexConfig; + } + + public function configure(): void + { + $this + ->setName('spameri:elastic:initialize-settings') + ->setDescription('Creates index and puts mapping and settings for entity/ies.') + ; + } + + + public function execute( + \Symfony\Component\Console\Input\InputInterface $input, + \Symfony\Component\Console\Output\OutputInterface $output + ) + { + $output->writeln(); + foreach ($this->indexConfig as $indexConfig) { + $output->writeln(); + $this->settinsCreator->create($indexConfig->provide()); + $output->writeln(); + } + + + $output->writeln(); + // foreach neon configs + $output->writeln(); + + + $output->writeln(); + // foreach annotation configs + $output->writeln(); + } + +} diff --git a/src/Commands/Migrate.php b/src/Commands/Migrate.php new file mode 100644 index 00000000..de166f54 --- /dev/null +++ b/src/Commands/Migrate.php @@ -0,0 +1,38 @@ +setName('spameri:elastic:migrate') + ->setDescription('Runs migrations from files.') + ->addArgument('filename', \Symfony\Component\Console\Input\InputArgument::OPTIONAL) + ; + } + + + protected function execute( + \Symfony\Component\Console\Input\InputInterface $input + , \Symfony\Component\Console\Output\OutputInterface $output + ) + { + $output->writeln('Starting'); + + // 1. Get folder + // 2. Iterate folder + // 3. Run each file in folder + // 3a. Check if file was executed - skip + // 3b. Check if file was changed - skip and report + // 4. Save executed files to ES + // 5. Done + + } +} diff --git a/src/Config/Elastic.neon b/src/Config/Elastic.neon index c6edb012..01bfb5fd 100644 --- a/src/Config/Elastic.neon +++ b/src/Config/Elastic.neon @@ -59,6 +59,9 @@ services: indicesCreate: class: Spameri\Elastic\Model\Indices\Create + indicesDelete: + class: Spameri\Elastic\Model\Indices\Delete + serviceLocator: class: Spameri\Elastic\Model\ServiceLocator @@ -75,7 +78,7 @@ services: class: Spameri\Elastic\ClientProvider elasticPanelLogger: - class: Spameri\Elastic\Diagnostics\PanelLogger(@elasticSearch.nullLogger) + class: Spameri\Elastic\Diagnostics\PanelLogger(@spameriElasticSearch.nullLogger) nullLogger: class: Psr\Log\NullLogger @@ -173,10 +176,28 @@ services: clientBuilder: class: Elasticsearch\ClientBuilder setup: - - setLogger(@elasticSearch.elasticPanelLogger) + - setLogger(@spameriElasticSearch.elasticPanelLogger) dateTimeProvider: - class: Spameri\Elastic\Provider\DateTimeProvider(@elasticSearch.dateTime) + class: Spameri\Elastic\Provider\DateTimeProvider(@spameriElasticSearch.dateTime) dateTime: class: \DateTimeImmutable + + nullOutput: + class: Symfony\Component\Console\Output\NullOutput + + consoleOutput: + class: Symfony\Component\Console\Output\ConsoleOutput + + nullLoggerHandler: + class: Spameri\Elastic\Import\Run\NullLoggerHandler + + nullLock: + class: Spameri\Elastic\Import\Lock\NullLock + + nullHandler: + class: Spameri\Elastic\Import\RunHandler\NullHandler + + nullAfterImport: + class: Spameri\Elastic\Import\AfterImport\NullAfterImport diff --git a/src/DI/ElasticSearchExtension.php b/src/DI/SpameriElasticSearchExtension.php similarity index 96% rename from src/DI/ElasticSearchExtension.php rename to src/DI/SpameriElasticSearchExtension.php index d3302713..423cabab 100644 --- a/src/DI/ElasticSearchExtension.php +++ b/src/DI/SpameriElasticSearchExtension.php @@ -3,7 +3,7 @@ namespace Spameri\Elastic\DI; -class ElasticSearchExtension extends \Nette\DI\CompilerExtension +class SpameriElasticSearchExtension extends \Nette\DI\CompilerExtension { public $defaults = [ @@ -100,7 +100,7 @@ public function loadConfiguration() : void $config = \Nette\DI\Config\Helpers::merge($this->getConfig(), $this->defaults); $config = $this->toggleSynonymAnalyzer($config); - $this->compiler->getContainerBuilder()->parameters['elasticSearch'] = $config; + $this->compiler->getContainerBuilder()->parameters['spameriElasticSearch'] = $config; $services = $this->loadFromFile(__DIR__ . '/../Config/Elastic.neon'); diff --git a/src/Diagnostics/PanelLogger.php b/src/Diagnostics/PanelLogger.php index 11b17c9b..f96c625f 100644 --- a/src/Diagnostics/PanelLogger.php +++ b/src/Diagnostics/PanelLogger.php @@ -148,6 +148,8 @@ private function logQuery( ) : void { if (isset($context['method'], $context['uri'])) { + $path = \explode('9200', $context['uri']); + $context['uri'] = $path[1] ?? $context['uri']; $this->queries[] = $context; } } diff --git a/src/Entity/AbstractImport.php b/src/Entity/AbstractImport.php new file mode 100644 index 00000000..8c703c2e --- /dev/null +++ b/src/Entity/AbstractImport.php @@ -0,0 +1,59 @@ +key = $key; + } + + + public function key() + { + return $this->key; + } + + + public function entityVariables(): array + { + $vars = \get_object_vars($this); + unset($vars['key']); + return $vars; + } + + + public function toArray(): array + { + $array = []; + + foreach ($this->entityVariables() as $key => $variable) { + if ($variable instanceof \Spameri\Elastic\Entity\Import\NoValue) { + continue; + } + + if ($variable instanceof \Spameri\Elastic\Entity\Import\ValidationPropertyInterface) { + $array[$variable->key()] = $variable->getValue(); + + } elseif ($variable instanceof \Spameri\Elastic\Entity\Import\ValidationObjectInterface) { + $array[$variable->key()] = $variable->toArray(); + + } else { + $array[$key] = $variable; + } + } + + return $array; + } + +} diff --git a/src/Entity/Import/ArrayValue.php b/src/Entity/Import/ArrayValue.php new file mode 100644 index 00000000..c02de347 --- /dev/null +++ b/src/Entity/Import/ArrayValue.php @@ -0,0 +1,40 @@ +array = $array; + $this->key = $key; + } + + + public function key(): string + { + return $this->key; + } + + + public function getValue(): array + { + return $this->array; + } + +} diff --git a/src/Entity/Import/BoolValue.php b/src/Entity/Import/BoolValue.php new file mode 100644 index 00000000..2d0b7f8d --- /dev/null +++ b/src/Entity/Import/BoolValue.php @@ -0,0 +1,41 @@ +value = $value; + $this->key = $key; + } + + + public function key(): string + { + return $this->key; + } + + + public function getValue(): bool + { + return $this->value; + } + +} diff --git a/src/Entity/Import/DateValue.php b/src/Entity/Import/DateValue.php new file mode 100644 index 00000000..151fa1f0 --- /dev/null +++ b/src/Entity/Import/DateValue.php @@ -0,0 +1,48 @@ +value = $value; + $this->key = $key; + $this->format = $format; + } + + + public function key(): string + { + return $this->key; + } + + + public function getValue(): string + { + return $this->value->format($this->format); + } + +} diff --git a/src/Entity/Import/EmptyValue.php b/src/Entity/Import/EmptyValue.php new file mode 100644 index 00000000..5edfca2c --- /dev/null +++ b/src/Entity/Import/EmptyValue.php @@ -0,0 +1,34 @@ +key = $key; + } + + + public function key(): string + { + return $this->key; + } + + + public function getValue() + { + return NULL; + } + +} diff --git a/src/Entity/Import/FloatValue.php b/src/Entity/Import/FloatValue.php new file mode 100644 index 00000000..79059ff2 --- /dev/null +++ b/src/Entity/Import/FloatValue.php @@ -0,0 +1,41 @@ +value = $value; + $this->key = $key; + } + + + public function key(): string + { + return $this->key; + } + + + public function getValue(): float + { + return $this->value; + } + +} diff --git a/src/Entity/Import/IntegerValue.php b/src/Entity/Import/IntegerValue.php new file mode 100644 index 00000000..fdad169a --- /dev/null +++ b/src/Entity/Import/IntegerValue.php @@ -0,0 +1,41 @@ +value = $value; + $this->key = $key; + } + + + public function key(): string + { + return $this->key; + } + + + public function getValue(): int + { + return $this->value; + } + +} diff --git a/src/Entity/Import/NoValue.php b/src/Entity/Import/NoValue.php new file mode 100644 index 00000000..1d52adf2 --- /dev/null +++ b/src/Entity/Import/NoValue.php @@ -0,0 +1,19 @@ +value = $value; + $this->key = $key; + } + + + public function key(): string + { + return $this->key; + } + + + public function getValue(): string + { + return $this->value; + } + +} diff --git a/src/Entity/Import/ValidationObjectInterface.php b/src/Entity/Import/ValidationObjectInterface.php new file mode 100644 index 00000000..0a0ea2d2 --- /dev/null +++ b/src/Entity/Import/ValidationObjectInterface.php @@ -0,0 +1,12 @@ +lockDir = $lockDir; + } + + + public function setRunName(string $runName): void + { + $this->runName = $runName; + } + + + public function acquire(int $time): \Spameri\Elastic\Import\LockInterface + { + $lockReleaseTime = \file_get_contents($this->getFileName()); + if ( + $lockReleaseTime + && $lockReleaseTime > \time() + ) { + throw new \Spameri\Elastic\Import\Exception\Fatal('Already locked.'); + } + + \file_put_contents($this->getFileName(), \time() + $time); + + return $this; + } + + + public function release(): void + { + \unlink($this->getFileName()); + } + + + public function extend(int $time): void + { + \file_put_contents($this->getFileName(), \time() + $time); + } + + + public function getFileName(): string + { + return $this->lockDir . \DIRECTORY_SEPARATOR . $this->runName; + } + +} diff --git a/src/Import/Lock/NullLock.php b/src/Import/Lock/NullLock.php new file mode 100644 index 00000000..d623c57b --- /dev/null +++ b/src/Import/Lock/NullLock.php @@ -0,0 +1,31 @@ +response = $response; + $this->entity = $entity; + } + + + public function isSuccessful(): bool + { + return $this->response ? TRUE : FALSE; + } + + + public function getResponse() + { + return $this->response; + } + + + public function getEntity($entity) + { + return $this->entity; + } + +} diff --git a/src/Import/ResponseInterface.php b/src/Import/ResponseInterface.php new file mode 100644 index 00000000..02c5c5a0 --- /dev/null +++ b/src/Import/ResponseInterface.php @@ -0,0 +1,19 @@ +output = $output; + $this->lock = $lock; + $this->loggerHandler = $loggerHandler; + $this->runHandler = $runHandler; + $this->dataProvider = $dataProvider; + $this->prepareImportData = $prepareImportData; + $this->dataImport = $dataImport; + $this->afterImport = $afterImport; + + $this->runName = (new \ReflectionClass($this))->getShortName(); + $this->lock->setRunName($this->runName); + $this->setUpLogger($logDir); + } + + + protected function setUpLogger($logDir): void + { + $directory = $logDir; + \Nette\Utils\FileSystem::createDir($directory); + + $directory .= \DIRECTORY_SEPARATOR; + $directory .= $this->runName; + \Nette\Utils\FileSystem::createDir($directory); + + $directory .= \DIRECTORY_SEPARATOR; + $directory .= (new \DateTime())->format('Y-m-d'); + \Nette\Utils\FileSystem::createDir($directory); + + $this->fileName = (new \DateTime())->format('H-i-s'); + } + + + public function execute( + \Spameri\Elastic\Import\Run\Options $options + ): void + { + $this->lock->acquire($options->lockDuration()); + + $this->initializeProgressBar($this->dataProvider->count($options)); + + $data = $this->dataProvider->provide($options); + + foreach ($data as $item) { + try { + $this->loggerHandler->logItemStart($item); + + $prepared = $this->prepareImportData->prepare($item); + $this->loggerHandler->logPrepared($prepared); + + $response = $this->dataImport->import($prepared); + $this->loggerHandler->logResponse($response); + + $this->afterImport->process($item, $response); + + $this->runHandler->advance($this->runName, $this->progressBar, $prepared); + + $this->lock->extend($options->lockDuration()); + + } catch (\Spameri\Elastic\Import\Exception\Omit $exception) { + $this->loggerHandler->logOmitException($exception); + + } catch (\Spameri\Elastic\Import\Exception\Error $exception) { + $this->loggerHandler->logErrorException($exception); + + } catch (\Spameri\Elastic\Import\Exception\Fatal $exception) { + $this->loggerHandler->logFatalException($exception); + } + } + + $this->runHandler->finish($this->runName, $this->progressBar, $prepared ?? NULL); + + $this->lock->release(); + + $this->loggerHandler->logFinish(); + } + + + public function initializeProgressBar(int $maxCount): void + { + $this->progressBar = new \Symfony\Component\Console\Helper\ProgressBar($this->output, $maxCount); + $this->progressBar->setFormat('debug'); + $this->progressBar->start($maxCount); + } + +} diff --git a/src/Import/Run/LoggerHandler.php b/src/Import/Run/LoggerHandler.php new file mode 100644 index 00000000..64b605db --- /dev/null +++ b/src/Import/Run/LoggerHandler.php @@ -0,0 +1,63 @@ +logger = $logger; + } + + + public function logItemStart($item): void + { + $this->logger->debug('Processing item ' . \Tracy\Dumper::toText($item)); + } + + + public function logPrepared(\Spameri\Elastic\Entity\AbstractImport $import): void + { + $this->logger->debug('Prepared data ' . \Tracy\Dumper::toText($import->toArray())); + } + + + public function logResponse(\Spameri\Elastic\Import\ResponseInterface $result): void + { + $this->logger->debug('Response ' . \Tracy\Dumper::toText($result)); + } + + + public function logOmitException(\Spameri\Elastic\Import\Exception\ImportException $exception): void + { + $this->logger->notice($exception->getMessage()); + } + + + public function logErrorException(\Spameri\Elastic\Import\Exception\ImportException $exception): void + { + $this->logger->error($exception->getMessage()); + } + + + public function logFatalException(\Spameri\Elastic\Import\Exception\ImportException $exception): void + { + $this->logger->critical($exception->getMessage()); + } + + + public function logFinish(): void + { + $this->logger->info('Finished'); + } + +} diff --git a/src/Import/Run/NullLoggerHandler.php b/src/Import/Run/NullLoggerHandler.php new file mode 100644 index 00000000..092dc44e --- /dev/null +++ b/src/Import/Run/NullLoggerHandler.php @@ -0,0 +1,49 @@ +lockDuration = $lockDuration; + } + + + public function lockDuration(): int + { + return $this->lockDuration; + } + +} diff --git a/src/Import/RunHandler/ConsoleHandler.php b/src/Import/RunHandler/ConsoleHandler.php new file mode 100644 index 00000000..bd71ef8b --- /dev/null +++ b/src/Import/RunHandler/ConsoleHandler.php @@ -0,0 +1,30 @@ +advance(); + if ($progressBar->getProgress() % 100) { + $progressBar->display(); + } + } + + + public function finish( + string $runName, + \Symfony\Component\Console\Helper\ProgressBar $progressBar, + ?\Spameri\Elastic\Entity\AbstractImport $lastProcessed + ): void + { + $progressBar->finish(); + } + +} diff --git a/src/Import/RunHandler/NullHandler.php b/src/Import/RunHandler/NullHandler.php new file mode 100644 index 00000000..991b8e21 --- /dev/null +++ b/src/Import/RunHandler/NullHandler.php @@ -0,0 +1,27 @@ +clientProvider = $clientProvider; + $this->output = $output; + } + + + public function before(): void + { + $this->output->writeln('Starting migration ' . self::class); + $this->output->writeln('Running before scripts'); + } + + + public function after(): void + { + $this->output->writeln('Running after scripts'); + $this->output->writeln('Finished migration ' . self::class); + } + +} diff --git a/src/Migration/MigrationFileInterface.php b/src/Migration/MigrationFileInterface.php new file mode 100644 index 00000000..32710dfe --- /dev/null +++ b/src/Migration/MigrationFileInterface.php @@ -0,0 +1,15 @@ +run($file); + } + } + + + public function run( + MigrationFileInterface $file + ): void + { + $file->before(); + $file->migrate(); + $file->after(); + } + +} diff --git a/src/Model/BaseService.php b/src/Model/BaseService.php index c34ab3fd..0cfa14fd 100644 --- a/src/Model/BaseService.php +++ b/src/Model/BaseService.php @@ -41,11 +41,6 @@ abstract class BaseService implements IService */ protected $getAllBy; - /** - * @var array - */ - protected $entityProperties; - /** * @var \Spameri\Elastic\Factory\IEntityFactory */ @@ -63,16 +58,15 @@ abstract class BaseService implements IService public function __construct( string $index - , array $entityProperties , \Spameri\Elastic\Factory\IEntityFactory $entityFactory , \Spameri\Elastic\Factory\ICollectionFactory $collectionFactory , \Spameri\Elastic\ClientProvider $client - , Insert $insert - , Get $get - , GetBy $getBy - , GetAllBy $getAllBy - , Delete $delete - , Aggregate $aggregate + , \Spameri\Elastic\Model\Insert $insert + , \Spameri\Elastic\Model\Get $get + , \Spameri\Elastic\Model\GetBy $getBy + , \Spameri\Elastic\Model\GetAllBy $getAllBy + , \Spameri\Elastic\Model\Delete $delete + , \Spameri\Elastic\Model\Aggregate $aggregate ) { $this->client = $client->client(); @@ -82,7 +76,6 @@ public function __construct( $this->delete = $delete; $this->getBy = $getBy; $this->getAllBy = $getAllBy; - $this->entityProperties = $entityProperties; $this->entityFactory = $entityFactory; $this->collectionFactory = $collectionFactory; $this->aggregate = $aggregate; diff --git a/src/Model/EntitySettingsLocator.php b/src/Model/EntitySettingsLocator.php new file mode 100644 index 00000000..990031dc --- /dev/null +++ b/src/Model/EntitySettingsLocator.php @@ -0,0 +1,35 @@ +indexConfig = $indexConfig; + } + + + public function locate($indexName) : \Spameri\ElasticQuery\Mapping\Settings + { + foreach ($this->indexConfig as $indexConfig) { + if ($indexConfig->provide()->indexName() === $indexName) { + return $indexConfig->provide(); + } + } + + throw new \Spameri\Elastic\Exception\SettingsNotLocated($indexName); + } + +} diff --git a/src/Model/Indices/Delete.php b/src/Model/Indices/Delete.php new file mode 100644 index 00000000..b0bae335 --- /dev/null +++ b/src/Model/Indices/Delete.php @@ -0,0 +1,44 @@ +clientProvider = $clientProvider; + } + + + public function execute( + string $index + ) : array + { + try { + /** @var array $result */ + $result = $this->clientProvider->client()->indices()->delete( + ( + new \Spameri\ElasticQuery\Document( + $index + ) + )->toArray() + ); + + return $result; + + } catch (\Elasticsearch\Common\Exceptions\ElasticsearchException $exception) { + throw new \Spameri\Elastic\Exception\ElasticSearch($exception->getMessage()); + } + } + +} diff --git a/src/Model/Insert/PrepareEntityArray.php b/src/Model/Insert/PrepareEntityArray.php index 645a8a93..cb0dbcbf 100644 --- a/src/Model/Insert/PrepareEntityArray.php +++ b/src/Model/Insert/PrepareEntityArray.php @@ -91,10 +91,14 @@ public function iterateVariables( \is_string($property) || \is_int($property) || \is_bool($property) + || \is_float($property) || $property === NULL ) { $preparedArray[$key] = $property; + } elseif (\is_array($property)) { + $preparedArray[$key] = $this->iterateVariables($property); + } elseif ($property instanceof \Spameri\Elastic\Entity\DateTimeInterface) { $preparedArray[$key] = $property->format(); diff --git a/src/Settings/IndexConfigInterface.php b/src/Settings/IndexConfigInterface.php new file mode 100644 index 00000000..3cfac8f2 --- /dev/null +++ b/src/Settings/IndexConfigInterface.php @@ -0,0 +1,10 @@ + 1, + 'name' => 'Avengers', + 'year' => 2019, + 'released' => TRUE, + ]); + $this->clientProvider->client()->index($document->toArray()); + } + +} diff --git a/tests/SpameriTests/Elastic/Migration/migrations/2019/09/24/Structure/Initialize.php b/tests/SpameriTests/Elastic/Migration/migrations/2019/09/24/Structure/Initialize.php new file mode 100644 index 00000000..ce179c52 --- /dev/null +++ b/tests/SpameriTests/Elastic/Migration/migrations/2019/09/24/Structure/Initialize.php @@ -0,0 +1,65 @@ +delete = $delete; + $this->create = $create; + $this->settings = $settings; + $this->mapping = $mapping; + } + + + public function before(): void + { + parent::before(); + $this->delete->execute('spameri'); + } + + + public function migrate(): void + { + $this->output->writeln('Creating mapping'); + $this->create->execute('spameri', []); + + $this->output->writeln('Put settings'); + $this->settings->execute('spameri', []); + + $this->output->writeln('Put mapping'); + $this->mapping->execute('spameri', []); + } + +} From d1b09ed8dd62a3bfb0d0d6db991d9aad0a88396c Mon Sep 17 00:00:00 2001 From: VBoss Date: Sat, 26 Oct 2019 15:03:20 +0200 Subject: [PATCH 05/87] bump ElasticQuery --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index e8b4fe4f..ebc7bc0a 100644 --- a/composer.json +++ b/composer.json @@ -29,7 +29,7 @@ "nette/di": "^2.4.10", "nette/security": "^2.3.0", "nette/utils": "^2.4.10", - "spameri/elastic-query": "^0.5.0", + "spameri/elastic-query": "^0.6.0", "elasticsearch/elasticsearch": "^6.0", "kdyby/console": "^2.7.1", "kdyby/datetime-provider": "v1.0.0", From eefb2b2fc8604e5dd6153cc977e0685808020f37 Mon Sep 17 00:00:00 2001 From: VBoss Date: Tue, 23 Apr 2019 01:52:45 +0200 Subject: [PATCH 06/87] Documentation --- doc/01_intro.md | 52 ++++++++--- doc/02_neon_configuration.md | 1 + doc/03_entity_class.md | 155 +++++++++++++++++++++++++------ doc/05_new_index_with_mapping.md | 20 ++++ doc/06_fill_data.md | 20 ++++ doc/07_save_explained.md | 5 + doc/08_basic_get.md | 11 +++ doc/09_match_get.md | 24 +++++ doc/10_aggregate.md | 20 ++++ doc/11_entity_factory.md | 26 ++++++ doc/12_entity_service.md | 65 +++++++++++++ doc/13_advanced_get.md | 21 +++++ 12 files changed, 379 insertions(+), 41 deletions(-) create mode 100644 doc/06_fill_data.md create mode 100644 doc/07_save_explained.md create mode 100644 doc/08_basic_get.md create mode 100644 doc/09_match_get.md create mode 100644 doc/10_aggregate.md create mode 100644 doc/11_entity_factory.md create mode 100644 doc/12_entity_service.md create mode 100644 doc/13_advanced_get.md diff --git a/doc/01_intro.md b/doc/01_intro.md index 57ebbdd2..24f5de22 100644 --- a/doc/01_intro.md +++ b/doc/01_intro.md @@ -11,7 +11,7 @@ Use composer `composer require spameri/elastic` In your config neon, enable extensions. Kdyby/Console is there because we need it to do some command line commands. Monolog is required by elasticsearch/elasticsearch and we can use existing extension in Kdyby/Monolog. -``` +```neon extensions: elasticSearch: \Spameri\Elastic\DI\ElasticSearchExtension console: Kdyby\Console\DI\ConsoleExtension @@ -19,41 +19,63 @@ extensions: ``` Then configure where is your ElasticSearch. -``` +```neon elasticSearch: host: 127.0.0.1 port: 9200 ``` -For more config options see default values in `\Spameri\Elastic\DI\ElasticSearchExtension::$defaults`. +For more config options see default values in `\Spameri\Elastic\DI\ElasticSearchExtension::$defaults`. [Here](../src/DI/ElasticSearchExtension.php#L9). + +--- ### 2. First entity -#### [Neon file configuration](../blob/master/doc/02_neon_configuration.md) +#### [Neon file configuration](02_neon_configuration.md) + +#### [Create entity class](03_entity_class.md) + +#### [Create entity service](12_entity_service.md) -#### [Create Entity class](../blob/master/doc/03_entity_class.md) +#### [Create entity factory](11_entity_factory.md) + +--- ### 3. Mapping -#### [Create new index with mapping]((../blob/master/doc/05_new_index_with_mapping.md)) +#### [Create new index with mapping](05_new_index_with_mapping.md) + +--- ### 4. Fill with data -TODO + +#### [Create and save entity](06_fill_data.md) + +#### [Saving process explained](07_save_explained.md) + +--- ### 5. Get data from ElasticSearch -TODO Tady factories -TODO -TODO -TODO -TODO + +#### [Get data by ID](08_basic_get.md) + +#### [Get data by tag](13_advanced_get.md) + +--- ### 6. Filter data from ElasticSearch -TODO + +#### [Match data](09_match_get.md) + +--- ### 7. Aggregate data from ElasticSearch -TODO + +#### [Aggregate data](10_aggregate.md) + +--- ### x. Other -#### [Data interfaces]((../blob/master/doc/04_data_interfaces.md)) +#### [Data interfaces](04_data_interfaces.md) diff --git a/doc/02_neon_configuration.md b/doc/02_neon_configuration.md index 38b9c280..c91e85a5 100644 --- a/doc/02_neon_configuration.md +++ b/doc/02_neon_configuration.md @@ -1,4 +1,5 @@ # Define structure in neon file +TODO convert to video example - Create neon file for example in `app/ProductModule/Config/Product.neon` - Import created file to your application config neon. Usually located in`app/config/config.neon`. diff --git a/doc/03_entity_class.md b/doc/03_entity_class.md index f471e647..a795ec6c 100644 --- a/doc/03_entity_class.md +++ b/doc/03_entity_class.md @@ -1,11 +1,11 @@ # Entity class -Lets create entity class, continuing our example, in folder `app/ProductModule/Entity/Product.php` given file contents: +Lets create entity class, continuing our example, in folder `tests/SpameriTests/Data/Entity/Video.php` given file contents: ```php -namespace App\ProductModule\Entity; +namespace SpameriTests\Data\Entity; -class Product implements \Spameri\Elastic\Entity\IElasticEntity +class Video implements \Spameri\Elastic\Entity\IElasticEntity { /** @@ -38,7 +38,7 @@ class Product implements \Spameri\Elastic\Entity\IElasticEntity ### Lets look at class part by part. -- Entity is in our defined namespace for `ProductModule` in own folder `Entity` which is shared for multiple entities. +- Entity is in our defined namespace in own folder `Entity` which is shared for multiple entities. - Class extends interface `\Spameri\Elastic\Entity\IElasticEntity`, this is core interface for ElasticSearch document. It has to have `id` provided by ElasticSearch, library takes care of handling this field, no need to add in mapping. - Based on this interface, library figures out how to save this class. @@ -47,26 +47,26 @@ It has to have `id` provided by ElasticSearch, library takes care of handling th - Interface requires function `id()` based on returned value it updates or creates entity. - Interface requires `entityVariables()` in this exact form. (This may be changed in future versions, but now is required) -### Adding properties to product entity +### Adding properties to video entity -#### Single value property - `name` +#### Single value property - `Video.Story.KeyWord` -- Lets say our product has limited name length to maximum of 255 characters and also 0 characters is not enough to -describe product. +- Lets say our Video has limited keyword length to maximum of 55 characters and also 0 characters is not enough to +describe keyword. - We do not have reliable data input so lets validate this with help of interface `\Spameri\Elastic\Entity\IValue`. -- First create value object for name `\App\ProductModule\Entity\Product\Name`. +- First create value object for keyword `\SpameriTests\Data\Entity\Video\Story\KeyWord`. - Class should implement interface `\Spameri\Elastic\Entity\IValue`. - `__construct` should have one parameter **string $value**. -- In construct do our validation for product name. +- In construct do our validation for keyword. - Implement `value()` method. -- Add property to `\App\ProductModule\Entity\Product` constructor. -- Generate getter in `\App\ProductModule\Entity\Product` entity. +- Add property to `\SpameriTests\Data\Entity\Video` constructor. +- Generate getter in `\SpameriTests\Data\Entity\Video` entity for KeyWord. - Result: ```php -namespace App\ProductModule\Entity\Product; +namespace SpameriTests\Data\Entity\Video\Story; -class Name implements \Spameri\Elastic\Entity\IValue +class KeyWord implements \Spameri\Elastic\Entity\IValue { /** @@ -79,11 +79,11 @@ class Name implements \Spameri\Elastic\Entity\IValue string $value ) { - if (\strlen($value) < 0) { - throw new \InvalidArgumentException('Empty string is not supported for product name: ' . $value); + if ($value === '') { + throw new \InvalidArgumentException(); } - if (\strlen($value) > 255) { - $value = \substr($value, 0, 255); + if (\strlen($value) > 55) { + throw new \InvalidArgumentException(); } $this->value = $value; @@ -98,14 +98,117 @@ class Name implements \Spameri\Elastic\Entity\IValue } ``` -#### Value collection property - `details.tags` -TODO +#### Value collection property - `Video.Story.KeyWordCollection` +TODO Description +```php +namespace SpameriTests\Data\Entity\Video\Story; + + +class KeyWordCollection implements \Spameri\Elastic\Entity\IValueCollection +{ + + /** + * @var array<\SpameriTests\Data\Entity\Video\Story\KeyWord> + */ + private $collection; + + + public function __construct( + KeyWord ... $entities + ) + { + $this->collection = []; + foreach ($entities as $keyWord) { + $this->add($keyWord); + } + } + + + public function add( + KeyWord $keyWord + ) : void + { + $this->collection[$keyWord->value()] = $keyWord; + } -#### Single entity property - `details` -TODO -#### Entity collection property - `parameterValues` -TODO + public function getIterator() : \ArrayIterator + { + return new \ArrayIterator($this->collection); + } +} +``` + +#### Single entity property - `Video.Story` +TODO Description +```php +namespace SpameriTests\Data\Entity\Video; + + +class Story implements \Spameri\Elastic\Entity\IEntity +{ + + /** + * @var \SpameriTests\Data\Entity\Video\Story\KeyWordCollection + */ + private $keyWords; + + + public function __construct( + \SpameriTests\Data\Entity\Video\Story\KeyWordCollection $keyWord + ) + { + $this->keyWords = $keyWord; + } + + + public function entityVariables() : array + { + return \get_object_vars($this); + } + + + public function key() : string + { + + } +} +``` + +#### Entity collection property - `Video.Connections.FollowsCollection` +TODO Description +```php +namespace SpameriTests\Data\Entity\Video\Connections; + + +class FollowsCollection extends \Spameri\Elastic\Entity\Collection\EntityCollection +{ + +} +``` + +#### ElasticEntity collection property - `Video.People` +TODO Description +```php +namespace SpameriTests\Data\Entity\Video; + + +class People extends \Spameri\Elastic\Entity\Collection\ElasticEntityCollection +{ + + public function personByImdb( + \SpameriTests\Data\Entity\Property\ImdbId $imdb + ) : ?\SpameriTests\Data\Entity\Person + { + /** @var \SpameriTests\Data\Entity\Person $entity */ + foreach ($this->collection() as $entity) { + if ($imdb->value() === $entity->identification()->imdb()->value()) { + return $entity; + } + } + + return NULL; + } +} +``` -#### ElasticEntity collection property - `details.accessories` -TODO diff --git a/doc/05_new_index_with_mapping.md b/doc/05_new_index_with_mapping.md index e69de29b..042dea37 100644 --- a/doc/05_new_index_with_mapping.md +++ b/doc/05_new_index_with_mapping.md @@ -0,0 +1,20 @@ +# Create index with mapping for entity + +Entity is configured (link) and defined in php (link) and next thing is to get these settings to ElasticSearch. + +## Index creating + +### New index - no previous data +TODO +```bash +php www/index spameri:elastic:create-index +``` + +### New index - deleting previous data +TODO + +### New index - preserving previous data +TODO + + + diff --git a/doc/06_fill_data.md b/doc/06_fill_data.md new file mode 100644 index 00000000..052dd32a --- /dev/null +++ b/doc/06_fill_data.md @@ -0,0 +1,20 @@ +# Create and save entity + +## Create +TODO +[Example](../tests/SpameriTests/Model/Insert.phpt#L16) +```php +$video = new \SpameriTests\Data\Entity\Video( + new \Spameri\Elastic\Entity\Property\EmptyElasticId(), + new \SpameriTests\Data\Entity\Video\Identification( + new \SpameriTests\Data\Entity\Property\ImdbId(4154796) + ) +); +``` + +## Save +TODO +```php +$videoService->insert($video); +``` + diff --git a/doc/07_save_explained.md b/doc/07_save_explained.md new file mode 100644 index 00000000..eb9dbdb8 --- /dev/null +++ b/doc/07_save_explained.md @@ -0,0 +1,5 @@ +# Insert explained + +- Converter +- ElasticSearch point of view +TODO \ No newline at end of file diff --git a/doc/08_basic_get.md b/doc/08_basic_get.md new file mode 100644 index 00000000..dbe5546a --- /dev/null +++ b/doc/08_basic_get.md @@ -0,0 +1,11 @@ +# Basic Get + +## Description +Basic get by id from ElasticSearch, with exact match. + +## Example +```php +$video = $videoService->get( + new \Spameri\Elastic\Entity\Property\ElasticId($id) +); +``` diff --git a/doc/09_match_get.md b/doc/09_match_get.md new file mode 100644 index 00000000..828eb8fc --- /dev/null +++ b/doc/09_match_get.md @@ -0,0 +1,24 @@ +# Filter data + +## Description +TODO + +## Example +```php +$elasticQuery = new \Spameri\ElasticQuery\ElasticQuery(); +$elasticQuery->query()->must()->add( + new \Spameri\ElasticQuery\Query\Range( + 'year', + 2018, + 2019 + ) +); +$elasticQuery->query()->must()->add( + new \Spameri\ElasticQuery\Query\Match( + 'name', + 'Avengers' + ) +); + +$video = $videoService->getBy($elasticQuery); +``` diff --git a/doc/10_aggregate.md b/doc/10_aggregate.md new file mode 100644 index 00000000..03249a47 --- /dev/null +++ b/doc/10_aggregate.md @@ -0,0 +1,20 @@ +# Aggregate + +## Description +TODO + +## Example +```php +$elasticQuery = new \Spameri\ElasticQuery\ElasticQuery(); +$elasticQuery->aggregation()->add( + new \Spameri\ElasticQuery\Aggregation\LeafAggregationCollection( + 'video-by-year', + NULL, + new \Spameri\ElasticQuery\Aggregation\Term( + 'year' + ) + ) +); + +$aggregateResult = $videoService->aggregate($elasticQuery); +``` diff --git a/doc/11_entity_factory.md b/doc/11_entity_factory.md new file mode 100644 index 00000000..bcac3609 --- /dev/null +++ b/doc/11_entity_factory.md @@ -0,0 +1,26 @@ +# Entity factory + +## Description +TODO +Creates entity from result hit. + +## Example +```php +namespace SpameriTests\Elastic\Factory; + + +class VideoFactory implements \Spameri\Elastic\Factory\IEntityFactory +{ + + public function create(\Spameri\ElasticQuery\Response\Result\Hit $hit) + { + return new \SpameriTests\Data\Entity\Video( + new \Spameri\Elastic\Entity\Property\ElasticId($hit->getValue('id')), + new \SpameriTests\Data\Entity\Video\Identification( + new \SpameriTests\Data\Entity\Property\ImdbId($hit->getValue('identification.imdb')) + ) + ); + } + +} +``` diff --git a/doc/12_entity_service.md b/doc/12_entity_service.md new file mode 100644 index 00000000..5c1d050c --- /dev/null +++ b/doc/12_entity_service.md @@ -0,0 +1,65 @@ +# Entity service + +## Description +TODO +Every service should extend BaseService which has all methods for entity manipulation. Like **Insert**, **Get**, +**GetBy**, **GetAllBy**. + +## Example +```php +namespace SpameriTests\Data\Model; + + +class VideoService extends \Spameri\Elastic\Model\BaseService +{ + + /** + * @param \Spameri\Elastic\Entity\IElasticEntity|\SpameriTests\Data\Entity\Video $entity + * @return string + */ + public function insert( + \Spameri\Elastic\Entity\IElasticEntity $entity + ) : string + { + return parent::insert($entity); + } + + + /** + * @param \Spameri\Elastic\Entity\Property\ElasticId $id + * @return \Spameri\Elastic\Entity\IElasticEntity|\SpameriTests\Data\Entity\Video + */ + public function get( + \Spameri\Elastic\Entity\Property\ElasticId $id + ) : \Spameri\Elastic\Entity\IElasticEntity + { + return parent::get($id); + } + + + /** + * @param \Spameri\ElasticQuery\ElasticQuery $elasticQuery + * @return \Spameri\Elastic\Entity\IElasticEntity|\SpameriTests\Data\Entity\Video + * @throws \Spameri\Elastic\Exception\DocumentNotFound + */ + public function getBy( + \Spameri\ElasticQuery\ElasticQuery $elasticQuery + ) : \Spameri\Elastic\Entity\IElasticEntity + { + return parent::getBy($elasticQuery); + } + + + /** + * @param \Spameri\ElasticQuery\ElasticQuery $elasticQuery + * @return \Spameri\Elastic\Entity\IElasticEntityCollection|array<\SpameriTests\Data\Entity\Video> + */ + public function getAllBy( + \Spameri\ElasticQuery\ElasticQuery $elasticQuery + ) : \Spameri\Elastic\Entity\IElasticEntityCollection + { + return parent::getAllBy($elasticQuery); + } + +} +``` diff --git a/doc/13_advanced_get.md b/doc/13_advanced_get.md new file mode 100644 index 00000000..47c832d9 --- /dev/null +++ b/doc/13_advanced_get.md @@ -0,0 +1,21 @@ +# Advanced Get + +## Description +TODO +Get data from ElasticSearch by tag. + +## Example +```php +$video = $videoService->getBy( + new \Spameri\ElasticQuery\ElasticQuery( + new \Spameri\ElasticQuery\Query\QueryCollection( + new \Spameri\ElasticQuery\Query\MustCollection( + new \Spameri\ElasticQuery\Query\Match( + 'story.tag', + 'action' + ) + ) + ) + ) +); +``` From d62925c5e3628d0b63b45541125c830e28785970 Mon Sep 17 00:00:00 2001 From: VBoss Date: Tue, 23 Apr 2019 17:38:23 +0200 Subject: [PATCH 07/87] Entity documentation + Raw usage documentation --- doc/01_intro.md | 21 +++ doc/03_entity_class.md | 230 ++++++++++++++++++++++++- src/Entity/AbstractEntity.php | 20 +++ src/Entity/AbstractValueCollection.php | 55 ++++++ 4 files changed, 320 insertions(+), 6 deletions(-) create mode 100644 src/Entity/AbstractEntity.php create mode 100644 src/Entity/AbstractValueCollection.php diff --git a/doc/01_intro.md b/doc/01_intro.md index 24f5de22..320d0d8b 100644 --- a/doc/01_intro.md +++ b/doc/01_intro.md @@ -27,6 +27,27 @@ elasticSearch: For more config options see default values in `\Spameri\Elastic\DI\ElasticSearchExtension::$defaults`. [Here](../src/DI/ElasticSearchExtension.php#L9). +#### Raw client usage +- After this configuration you are ready to use ElasticSearch in your Nette application. +- Where needed just inject `\Spameri\Elastic\ClientProvider` and then directly call what you need, like this: +```php +$result = $this->clientProvider->client()->search( + ( + new \Spameri\ElasticQuery\Document( + $index, + new \Spameri\ElasticQuery\Document\Body\Plain( + $elasticQuery->toArray() + ), + $index + ) + )->toArray() +); +``` +- [Client](https://github.com/elastic/elasticsearch-php/blob/master/src/Elasticsearch/Client.php) is provided from **elasticsearch/elasticsearch** and you can see their [documentation](https://github.com/elastic/elasticsearch-php#quickstart) +what methods and arrays are supported. +- When in doubt what how many arrays or how many arguments **match** supports use [Spameri/ElasticQuery](https://github.com/Spameri/ElasticQuery/blob/master/doc/02-query-objects.md) +- This is library used in later examples. But direct approach is also possible. + --- ### 2. First entity diff --git a/doc/03_entity_class.md b/doc/03_entity_class.md index a795ec6c..524e53b4 100644 --- a/doc/03_entity_class.md +++ b/doc/03_entity_class.md @@ -99,7 +99,13 @@ class KeyWord implements \Spameri\Elastic\Entity\IValue ``` #### Value collection property - `Video.Story.KeyWordCollection` -TODO Description +- If you need array of scalar values lets create ValueCollection. +- For easy setup you can use `\Spameri\Elastic\Entity\AbstractValueCollection` just create your collection and extend this abstract as you need. +For more advanced and typed approach use interface, as described next. +- Interface `\Spameri\Elastic\Entity\IValueCollection` is when you want typed and validated scalar value collection. +- After implementing interface you need to implement **getIterator()** method. +- Next to be type save you want to add methods **add**, **remove**, **get**, **__construct** +- For **__construct** you best fill values to collection as here [\Spameri\Elastic\Entity\AbstractValueCollection#L20](../src/Entity/AbstractValueCollection.php#L20) ```php namespace SpameriTests\Data\Entity\Video\Story; @@ -125,13 +131,28 @@ class KeyWordCollection implements \Spameri\Elastic\Entity\IValueCollection public function add( - KeyWord $keyWord + \SpameriTests\Data\Entity\Video\Story\KeyWord $keyWord ) : void { $this->collection[$keyWord->value()] = $keyWord; } + public function remove(string $key) : void + { + unset($this->collection[$key]); + } + + + public function get(string $key) : ?\SpameriTests\Data\Entity\Video\Story\KeyWord + { + if ( ! isset($this->collection[$key])) { + return NULL; + } + + return $this->collection[$key]; + } + public function getIterator() : \ArrayIterator { return new \ArrayIterator($this->collection); @@ -140,7 +161,11 @@ class KeyWordCollection implements \Spameri\Elastic\Entity\IValueCollection ``` #### Single entity property - `Video.Story` -TODO Description +- If you need some nested structure `\Spameri\Elastic\Entity\IEntity` interface is here for you. +- Also when feeling lazy there is `\Spameri\Elastic\Entity\AbstractEntity` for you to extend with methods implemented. +- In our example we have entity **Story** to encapsulate keywords and other story related properties. +- Library then can convert this entity to array and save it as array with no more help. + ```php namespace SpameriTests\Data\Entity\Video; @@ -170,13 +195,15 @@ class Story implements \Spameri\Elastic\Entity\IEntity public function key() : string { - + return \md5(\implode('_', $this->entityVariables())); } } ``` #### Entity collection property - `Video.Connections.FollowsCollection` -TODO Description +- ElasticSearch is powerful tool and it allows you to nest objects and collection as you need, so you can make collection of nested objects. +- This is simple you have Entity **Story** with implemented `IEntity` interface and all you need is create collection, extend `class FollowsCollection extends \Spameri\Elastic\Entity\Collection\EntityCollection` +and you are done. ```php namespace SpameriTests\Data\Entity\Video\Connections; @@ -188,7 +215,12 @@ class FollowsCollection extends \Spameri\Elastic\Entity\Collection\EntityCollect ``` #### ElasticEntity collection property - `Video.People` -TODO Description +- `\Spameri\Elastic\Entity\Collection\ElasticEntityCollection` provides basic relations for entities in ElasticSearch. +- It saves **_id** to current entity as reference in raw data but when loaded you have full entity with that id. +Any changes made to related entity/ies will be persisted when main entity is saved. +- Entity can be manually related 1:1 with manual lazy load in Factory (example in [factory](11_entity_factory.md) documentation) +- Or multiple entities can be in collection lazily loaded all at once, also in factory example. +- All you need is extend `\Spameri\Elastic\Entity\Collection\ElasticEntityCollection` and fill with your entities, library will do saving and resolving for you. ```php namespace SpameriTests\Data\Entity\Video; @@ -212,3 +244,189 @@ class People extends \Spameri\Elastic\Entity\Collection\ElasticEntityCollection } ``` +## Final product [Example](../tests/SpameriTests/Data/Entity/Video.php) +```php +namespace SpameriTests\Data\Entity; + + +class Video implements \Spameri\Elastic\Entity\IElasticEntity +{ + + /** + * @var \Spameri\Elastic\Entity\Property\IElasticId + */ + private $id; + + /** + * @var \SpameriTests\Data\Entity\Video\Identification + */ + private $identification; + + /** + * @var \SpameriTests\Data\Entity\Property\Name + */ + private $name; + + /** + * @var \SpameriTests\Data\Entity\Property\Year + */ + private $year; + + /** + * @var \SpameriTests\Data\Entity\Video\Technical + */ + private $technical; + + /** + * @var \SpameriTests\Data\Entity\Video\Story + */ + private $story; + + /** + * @var \SpameriTests\Data\Entity\Video\Details + */ + private $details; + + /** + * @var \SpameriTests\Data\Entity\Video\HighLights + */ + private $highLights; + + /** + * @var \SpameriTests\Data\Entity\Video\Connections + */ + private $connections; + + /** + * @var \SpameriTests\Data\Entity\Video\SeasonCollection + */ + private $season; + + /** + * @var \SpameriTests\Data\Entity\Video\People + */ + private $people; + + + public function __construct( + \Spameri\Elastic\Entity\Property\IElasticId $id + , \SpameriTests\Data\Entity\Video\Identification $identification + , \SpameriTests\Data\Entity\Property\Name $name + , \SpameriTests\Data\Entity\Property\Year $year + , \SpameriTests\Data\Entity\Video\Technical $technical + , \SpameriTests\Data\Entity\Video\Story $story + , \SpameriTests\Data\Entity\Video\Details $details + , \SpameriTests\Data\Entity\Video\HighLights $highLights + , \SpameriTests\Data\Entity\Video\Connections $connections + , \SpameriTests\Data\Entity\Video\People $people + , \SpameriTests\Data\Entity\Video\SeasonCollection $season = NULL + ) + { + $this->id = $id; + $this->identification = $identification; + $this->name = $name; + $this->year = $year; + $this->technical = $technical; + $this->story = $story; + $this->details = $details; + $this->highLights = $highLights; + $this->connections = $connections; + + if ($season === NULL) { + $season = new \SpameriTests\Data\Entity\Video\SeasonCollection(); + } + $this->season = $season; + $this->people = $people; + } + + + public function entityVariables() : array + { + return \get_object_vars($this); + } + + + public function id() : \Spameri\Elastic\Entity\Property\IElasticId + { + return $this->id; + } + + + public function identification() : \SpameriTests\Data\Entity\Video\Identification + { + return $this->identification; + } + + + public function name() : \SpameriTests\Data\Entity\Property\Name + { + return $this->name; + } + + + public function rename(\SpameriTests\Data\Entity\Property\Name $name) : void + { + $this->name = $name; + } + + + public function year() : \SpameriTests\Data\Entity\Property\Year + { + return $this->year; + } + + + public function setYear(\SpameriTests\Data\Entity\Property\Year $year) : void + { + $this->year = $year; + } + + + public function technical() : \SpameriTests\Data\Entity\Video\Technical + { + return $this->technical; + } + + + public function setTechnicalFromImdb(\SpameriTests\Data\Entity\Video\Technical $technical) : void + { + $this->technical = $technical; + } + + + public function story() : \SpameriTests\Data\Entity\Video\Story + { + return $this->story; + } + + + public function details() : \SpameriTests\Data\Entity\Video\Details + { + return $this->details; + } + + + public function highLights() : \SpameriTests\Data\Entity\Video\HighLights + { + return $this->highLights; + } + + + public function connections() : \SpameriTests\Data\Entity\Video\Connections + { + return $this->connections; + } + + + public function season() : \SpameriTests\Data\Entity\Video\SeasonCollection + { + return $this->season; + } + + + public function people() : \SpameriTests\Data\Entity\Video\People + { + return $this->people; + } +} +``` diff --git a/src/Entity/AbstractEntity.php b/src/Entity/AbstractEntity.php new file mode 100644 index 00000000..d0aef412 --- /dev/null +++ b/src/Entity/AbstractEntity.php @@ -0,0 +1,20 @@ +entityVariables())); + } + +} diff --git a/src/Entity/AbstractValueCollection.php b/src/Entity/AbstractValueCollection.php new file mode 100644 index 00000000..57265e8c --- /dev/null +++ b/src/Entity/AbstractValueCollection.php @@ -0,0 +1,55 @@ + + */ + protected $collection; + + + public function __construct( + \Spameri\Elastic\Entity\IValue ... $collection + ) + { + $this->collection = []; + foreach ($collection as $value) { + $this->add($value); + } + } + + + public function add( + \Spameri\Elastic\Entity\IValue $value + ) : void + { + $this->collection[$value->value()] = $value; + } + + + public function remove($key) : void + { + unset($this->collection[$key]); + } + + + public function get($key) : ?\Spameri\Elastic\Entity\IValue + { + if ( ! isset($this->collection[$key])) { + return NULL; + } + + return $this->collection[$key]; + } + + + public function getIterator() : \ArrayIterator + { + return new \ArrayIterator($this->collection); + } + +} From 55956ca1d339748d7978e79e5c44c0e8a6bde891 Mon Sep 17 00:00:00 2001 From: VBoss Date: Wed, 24 Apr 2019 00:46:27 +0200 Subject: [PATCH 08/87] Documentation --- doc/05_new_index_with_mapping.md | 43 ++++++++++++++++++++----- doc/06_fill_data.md | 15 ++++++--- doc/07_save_explained.md | 42 +++++++++++++++++++++++-- doc/09_match_get.md | 6 ++-- doc/10_aggregate.md | 4 ++- doc/13_advanced_get.md | 54 ++++++++++++++++++++++++++------ 6 files changed, 136 insertions(+), 28 deletions(-) diff --git a/doc/05_new_index_with_mapping.md b/doc/05_new_index_with_mapping.md index 042dea37..da64332c 100644 --- a/doc/05_new_index_with_mapping.md +++ b/doc/05_new_index_with_mapping.md @@ -1,20 +1,49 @@ # Create index with mapping for entity -Entity is configured (link) and defined in php (link) and next thing is to get these settings to ElasticSearch. +Entity is configured [link](03_entity_class.md) and defined in php [link](03_entity_class.md#L247) and next thing is to get these settings to ElasticSearch. ## Index creating -### New index - no previous data -TODO +### Variant 1. - No previous data +- When to use? +- You dont have any index, your ElasticSearch installation is clean and with no index for our entities. +- How? +- Simple neon is configured from previous [step](02_neon_configuration.md) and we just need to run command. + ```bash php www/index spameri:elastic:create-index ``` -### New index - deleting previous data -TODO +- Can be used for specific entity +```bash +php www/index spameri:elastic:create-index video +``` -### New index - preserving previous data -TODO +### Variant 2. - Deleting previous data +- When to use? +- You have existing index with outdated data with old configuration, but you dont need to keep them. This is +usually when you are generating ElasticSearch data from another database. +- How? +- Just add -f option. First thing command does is delete old index, then creating new empty index with new settings. +```bash +php www/index spameri:elastic:create-index -f +``` +- Can be used for specific entity +```bash +php www/index spameri:elastic:create-index -f video +``` +### Variant 3. - Preserving previous data +- When to use? +- You dont have source for data saved in ElasticSearch. +- Best is backup your data with command. This creates bulk json document with all data from index. +```bash +php www/index spameri:elastic:dump-index +``` +- With data backed up, now you can delete index and create it fresh with new mapping - following variant 2. +- Last thing is get your old data to new index wit this command. +```bash +php www/index spameri:elastic:restore-index +``` diff --git a/doc/06_fill_data.md b/doc/06_fill_data.md index 052dd32a..ecc1989c 100644 --- a/doc/06_fill_data.md +++ b/doc/06_fill_data.md @@ -1,20 +1,25 @@ # Create and save entity ## Create -TODO -[Example](../tests/SpameriTests/Model/Insert.phpt#L16) +- When creating new entity, mostly you need to construct it manually. +- Your data is from another database or API or csv or wherever. Quality may wary. +- So best approach is to validate all you can with objects. +- In this example we have property **imdb** and it needs to be in range from one digit to ten digits, rather than adding +if conditions wherever we create entity we use **ImdbId** class to validate this rule and this helps to properly convert +entity data to array. +- [Example](../tests/SpameriTests/Model/Insert.phpt#L16) ```php +$sqlData = $dibi->fetchRow(); $video = new \SpameriTests\Data\Entity\Video( new \Spameri\Elastic\Entity\Property\EmptyElasticId(), new \SpameriTests\Data\Entity\Video\Identification( - new \SpameriTests\Data\Entity\Property\ImdbId(4154796) + new \SpameriTests\Data\Entity\Property\ImdbId($sqlData['imdb']) ) ); ``` ## Save -TODO +- Entity is created, validated and ready to be saved. Just pass entity to [service](12_entity_service.md). And done. ```php $videoService->insert($video); ``` - diff --git a/doc/07_save_explained.md b/doc/07_save_explained.md index eb9dbdb8..047564b0 100644 --- a/doc/07_save_explained.md +++ b/doc/07_save_explained.md @@ -1,5 +1,41 @@ # Insert explained -- Converter -- ElasticSearch point of view -TODO \ No newline at end of file +## \Spameri\Elastic\Model\Insert\PrepareEntityArray +- This class is responsible for converting ElasticSearch entity to array that can be then saved to ElasticSearch. +- Configuring is done by implementing [interfaces](04_data_interfaces.md), no need for annotations or neon. + +### ::prepare(\Spameri\Elastic\Entity\IElasticEntity $entity) +- This method is here for last before convert modifications. +- If implemented `\Spameri\Elastic\Entity\ITrackedEntity` interface for entity tracking (and properties specified), +this method adds timestamp and user who edited/created entity. +- Then calling iterateVariables to prepare rest of entity array. + +### ::iterateVariables(array $variables) +This method accepts entity variables and based on their type performs converting to array to ready entity for insert. +There are 9 types of property handled: +1. **\Spameri\Elastic\Entity\IElasticEntity** in this case service locator comes to play and locates service for related +entity and saves connected entity. And prepares related entity's id to property. Because to ElasticSearch goes only +string id. + +2. **\Spameri\Elastic\Entity\IEntity** this is structural entity and is saved directly to parent entity. Its properties +are iterated with `::iterateVariables($property->entityVariables())` + +3. **\Spameri\Elastic\Entity\IValue** raw value in object, directly converted to array. + +4. **\Spameri\Elastic\Entity\IEntityCollection** Collection of structural class **IEntity**, iterate and act as step 2. + +5. **\Spameri\Elastic\Entity\IElasticEntityCollection** Collection of ElasticSearch entities **IElasticEntity**, +iterate and act as step 1. + +6. **\Spameri\Elastic\Entity\IValueCollection** Collection of **IValue**, iterate and act as step 3. + +7. Scalar values **string**, **int**, **bool** or **NULL**, no action just pass to array. + +8. **\Spameri\Elastic\Entity\DateTimeInterface** Date interface with specified format by this library so ElasticSearch +can save it without problems. +- **\Spameri\Elastic\Entity\Property\Date** for `Y-m-d` format +- **\Spameri\Elastic\Entity\Property\DateTime** for `Y-m-d\TH:i:s` format + +9. **\DateTime** All other Dates are converted to `Y-m-d\TH:i:s` + +10. Exception thrown property is none of above. diff --git a/doc/09_match_get.md b/doc/09_match_get.md index 828eb8fc..83752720 100644 --- a/doc/09_match_get.md +++ b/doc/09_match_get.md @@ -1,15 +1,17 @@ # Filter data ## Description -TODO +You can specify complicated ElasticSearch Query and still get pretty entity. [Service](12_entity_service.md) accepts +`\Spameri\ElasticQuery\ElasticQuery` object and returns entity or collection depending if you want one or more results. ## Example +In this example we are looking for video named 'Avengers' made in years from 2017 to 2018. ```php $elasticQuery = new \Spameri\ElasticQuery\ElasticQuery(); $elasticQuery->query()->must()->add( new \Spameri\ElasticQuery\Query\Range( 'year', - 2018, + 2017, 2019 ) ); diff --git a/doc/10_aggregate.md b/doc/10_aggregate.md index 03249a47..34cccbe6 100644 --- a/doc/10_aggregate.md +++ b/doc/10_aggregate.md @@ -1,9 +1,11 @@ # Aggregate ## Description -TODO +When aggregating you dont get directly entity just array data, for this we have `\Spameri\ElasticQuery\Response\ResultSearch` +object to encapsulate result. ## Example +In this example we want number of videos released in each year. ```php $elasticQuery = new \Spameri\ElasticQuery\ElasticQuery(); $elasticQuery->aggregation()->add( diff --git a/doc/13_advanced_get.md b/doc/13_advanced_get.md index 47c832d9..fac65c13 100644 --- a/doc/13_advanced_get.md +++ b/doc/13_advanced_get.md @@ -1,21 +1,55 @@ # Advanced Get ## Description -TODO -Get data from ElasticSearch by tag. + ## Example +In this example we want videos with tag **action** and are public. Also we want only first 50 sorted by year, but if +year has multiple videos sort them by score. Bonus points if movie is on beach with someone named john or here +misspelled as jon. + +Query does not have to be specified all at once, this is demonstration of capabilities. Query can be constructed empty +and extended through application runtime. ```php -$video = $videoService->getBy( - new \Spameri\ElasticQuery\ElasticQuery( - new \Spameri\ElasticQuery\Query\QueryCollection( - new \Spameri\ElasticQuery\Query\MustCollection( - new \Spameri\ElasticQuery\Query\Match( - 'story.tag', - 'action' - ) +$elasticQuery = new \Spameri\ElasticQuery\ElasticQuery( + new \Spameri\ElasticQuery\Query\QueryCollection( + new \Spameri\ElasticQuery\Query\MustCollection( + new \Spameri\ElasticQuery\Query\Term( + 'story.tag', + 'action' + ), + new \Spameri\ElasticQuery\Query\Term( + 'isPublic', + TRUE + ) + ), + new \Spameri\ElasticQuery\Query\ShouldCollection( + new \Spameri\ElasticQuery\Query\Match( + 'story.description', + 'beach' + ), + new \Spameri\ElasticQuery\Query\Fuzzy( + 'story.description', + 'jon' + ) + ) + ), + NULL, + NULL, + new \Spameri\ElasticQuery\Options( + 50, + NULL, + new \Spameri\ElasticQuery\Options\SortCollection( + new \Spameri\ElasticQuery\Options\Sort( + 'year', + \Spameri\ElasticQuery\Options\Sort::DESC + ), + new \Spameri\ElasticQuery\Options\Sort( + '_score', + \Spameri\ElasticQuery\Options\Sort::DESC ) ) ) ); +$videos = $this->videoService->getAllBy($elasticQuery); ``` From 63f5269c8fba511d95f12b6d220661d0b4048588 Mon Sep 17 00:00:00 2001 From: VBoss Date: Wed, 24 Apr 2019 15:53:37 +0200 Subject: [PATCH 09/87] BoolValue, IntegerValue, StringValue and NullValue - as prepared implemented strict value objects - still preferable to extend these and add your specific validation and name to differentiate values in your application --- src/Entity/Value/BoolValue.php | 28 +++++++++++++++++++++++++ src/Entity/Value/IntegerValue.php | 28 +++++++++++++++++++++++++ src/Entity/Value/NullValue.php | 28 +++++++++++++++++++++++++ src/Entity/Value/StringValue.php | 28 +++++++++++++++++++++++++ src/Model/Insert/PrepareEntityArray.php | 2 +- 5 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 src/Entity/Value/BoolValue.php create mode 100644 src/Entity/Value/IntegerValue.php create mode 100644 src/Entity/Value/NullValue.php create mode 100644 src/Entity/Value/StringValue.php diff --git a/src/Entity/Value/BoolValue.php b/src/Entity/Value/BoolValue.php new file mode 100644 index 00000000..94550513 --- /dev/null +++ b/src/Entity/Value/BoolValue.php @@ -0,0 +1,28 @@ +value = $value; + } + + + public function value(): bool + { + return $this->value; + } + +} diff --git a/src/Entity/Value/IntegerValue.php b/src/Entity/Value/IntegerValue.php new file mode 100644 index 00000000..ca8eea47 --- /dev/null +++ b/src/Entity/Value/IntegerValue.php @@ -0,0 +1,28 @@ +value = $value; + } + + + public function value(): int + { + return $this->value; + } + +} diff --git a/src/Entity/Value/NullValue.php b/src/Entity/Value/NullValue.php new file mode 100644 index 00000000..ae308024 --- /dev/null +++ b/src/Entity/Value/NullValue.php @@ -0,0 +1,28 @@ +value; + } + +} diff --git a/src/Entity/Value/StringValue.php b/src/Entity/Value/StringValue.php new file mode 100644 index 00000000..f0f9ae11 --- /dev/null +++ b/src/Entity/Value/StringValue.php @@ -0,0 +1,28 @@ +value = $value; + } + + + public function value(): string + { + return $this->value; + } + +} diff --git a/src/Model/Insert/PrepareEntityArray.php b/src/Model/Insert/PrepareEntityArray.php index cb0dbcbf..18859e21 100644 --- a/src/Model/Insert/PrepareEntityArray.php +++ b/src/Model/Insert/PrepareEntityArray.php @@ -76,7 +76,7 @@ public function iterateVariables( } elseif ($property instanceof \Spameri\Elastic\Entity\IValueCollection) { $preparedArray[$key] = []; - /** @var $value \Spameri\Elastic\Entity\IValue */ + /** @var \Spameri\Elastic\Entity\IValue $value */ /** @var \Spameri\Elastic\Entity\IValueCollection $property */ foreach ($property as $value) { if ($value instanceof \Spameri\Elastic\Entity\IValue) { From 98de222132eda7a0ba9550c1824570af9daed5d1 Mon Sep 17 00:00:00 2001 From: VBoss Date: Fri, 25 Oct 2019 16:03:31 +0200 Subject: [PATCH 10/87] QuickStart guide - Register extension --- doc/00_quick_start.md | 35 ++++++++++++++++++++++ doc/01_intro.md | 6 ++-- doc/02_neon_configuration.md | 8 ++--- tests/SpameriTests/Data/Config/Common.neon | 4 +-- tests/SpameriTests/Data/Config/Person.neon | 2 +- tests/SpameriTests/Data/Config/Video.neon | 2 +- 6 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 doc/00_quick_start.md diff --git a/doc/00_quick_start.md b/doc/00_quick_start.md new file mode 100644 index 00000000..89272dd9 --- /dev/null +++ b/doc/00_quick_start.md @@ -0,0 +1,35 @@ +# Quick Start + +## 1. Install + +Use composer to install this library. +```bash +composer require spameri/elastic +``` + +## 2. Configure + +You need to set up few things first, before you can dive into ElasticSearch. + +### I. Register extension + +In your configuration neon file you need to add these lines to `extension:` section. + +```yaml +extensions: + spameriElasticSearch: \Spameri\Elastic\DI\SpameriElasticSearchExtension + console: Kdyby\Console\DI\ConsoleExtension + monolog: Kdyby\Monolog\DI\MonologExtension +``` + + + + + + + + + + + + diff --git a/doc/01_intro.md b/doc/01_intro.md index 320d0d8b..ea9e501b 100644 --- a/doc/01_intro.md +++ b/doc/01_intro.md @@ -13,19 +13,19 @@ Monolog is required by elasticsearch/elasticsearch and we can use existing exten ```neon extensions: - elasticSearch: \Spameri\Elastic\DI\ElasticSearchExtension + spameriElasticSearch: \Spameri\Elastic\DI\SpameriElasticSearchExtension console: Kdyby\Console\DI\ConsoleExtension monolog: Kdyby\Monolog\DI\MonologExtension ``` Then configure where is your ElasticSearch. ```neon -elasticSearch: +spameriElasticSearch: host: 127.0.0.1 port: 9200 ``` -For more config options see default values in `\Spameri\Elastic\DI\ElasticSearchExtension::$defaults`. [Here](../src/DI/ElasticSearchExtension.php#L9). +For more config options see default values in `\Spameri\Elastic\DI\SpameriElasticSearchExtension::$defaults`. [Here](../src/DI/ElasticSearchExtension.php#L9). #### Raw client usage - After this configuration you are ready to use ElasticSearch in your Nette application. diff --git a/doc/02_neon_configuration.md b/doc/02_neon_configuration.md index c91e85a5..91e158b5 100644 --- a/doc/02_neon_configuration.md +++ b/doc/02_neon_configuration.md @@ -13,7 +13,7 @@ have to worry about it because of type deprecation. More details here https://ww - Entity definition is in neon under namespace `elasticSearch.entities.EntityName` continuing our example in file `app/ProductModule/Config/Product.neon`: ```neon -elasticSearch: +spameriElasticSearch: entities: Product: index: shop_product @@ -25,7 +25,7 @@ This means newly introduced fields not specified in mapping will throw error whe all fields introduced and specify their type. But if your application can add fields as needed you need to remember this strict limitation or just do not enable it. ```neon -elasticSearch: +spameriElasticSearch: entities: Product: dynamic: strict @@ -33,7 +33,7 @@ elasticSearch: - Now to specify entity mapping. Each object or encapsulation of sub fields stars with `properties:` then property name and under it you can specify type and analyzer. ```neon -elasticSearch: +spameriElasticSearch: entities: Product: properties: @@ -46,7 +46,7 @@ elasticSearch: - ElasticSearch default analyzers: https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-analyzers.html - Subfields example: ```neon -elasticSearch: +spameriElasticSearch: entities: Product: properties: diff --git a/tests/SpameriTests/Data/Config/Common.neon b/tests/SpameriTests/Data/Config/Common.neon index f68cf829..66886430 100644 --- a/tests/SpameriTests/Data/Config/Common.neon +++ b/tests/SpameriTests/Data/Config/Common.neon @@ -1,7 +1,7 @@ extensions: - elasticSearch: Spameri\Elastic\DI\ElasticSearchExtension + spameriElasticSearch: Spameri\Elastic\DI\SpameriElasticSearchExtension -elasticSearch: +spameriElasticSearch: host: 127.0.0.1 port: 9200 diff --git a/tests/SpameriTests/Data/Config/Person.neon b/tests/SpameriTests/Data/Config/Person.neon index 06e07011..44bc5149 100644 --- a/tests/SpameriTests/Data/Config/Person.neon +++ b/tests/SpameriTests/Data/Config/Person.neon @@ -1,4 +1,4 @@ -elasticSearch: +spameriElasticSearch: entities: Person: index: spameri_person diff --git a/tests/SpameriTests/Data/Config/Video.neon b/tests/SpameriTests/Data/Config/Video.neon index cffbe5de..c6015abb 100644 --- a/tests/SpameriTests/Data/Config/Video.neon +++ b/tests/SpameriTests/Data/Config/Video.neon @@ -1,4 +1,4 @@ -elasticSearch: +spameriElasticSearch: entities: Video: index: spameri_video From 8fbcd2415fb7c1c9611a1ead8060d3531c810977 Mon Sep 17 00:00:00 2001 From: VBoss Date: Fri, 25 Oct 2019 16:09:09 +0200 Subject: [PATCH 11/87] QuickStart guide - Configure ES url --- doc/00_quick_start.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/00_quick_start.md b/doc/00_quick_start.md index 89272dd9..55f3e3ce 100644 --- a/doc/00_quick_start.md +++ b/doc/00_quick_start.md @@ -22,9 +22,19 @@ extensions: monolog: Kdyby\Monolog\DI\MonologExtension ``` +### II. Configure +Now you need to tell library where is ElasticSearch running. Default values are **localhost** +and port **9200**. That means if you are running ElasticSearch locally with default port, no +need to configure anything. +```yaml +spameriElasticSearch: + host: 192.168.0.14 + port: 9200 +``` +### III. From cf77334e75e81fcf3a6f85b775082ea9812d014b Mon Sep 17 00:00:00 2001 From: VBoss Date: Fri, 25 Oct 2019 16:34:01 +0200 Subject: [PATCH 12/87] QuickStart guide - Configure Entity --- doc/00_quick_start.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/doc/00_quick_start.md b/doc/00_quick_start.md index 55f3e3ce..bf446a45 100644 --- a/doc/00_quick_start.md +++ b/doc/00_quick_start.md @@ -34,10 +34,28 @@ spameriElasticSearch: port: 9200 ``` -### III. - +### III. Configure Entity +Next step is to configure your first entity. This entity is for e-shop product. +```yaml +1.| spameriElasticSearch: +2.| entities: +3.| SimpleProduct: +4.| index: spameri_simple_product +5.| dynamic: strict +6.| config: @simpleProductConfig +7.| properties: +``` +- First line is extensionName +- Second line is entities config array +- Third line is EntityName +- Fourth line is index name for this entity +- Fifth line is for specifying whether index should accept new not specified fields +- Sixth line is reference to where is object with entity configuration +- Seventh line is where you can configure your entity within this neon + +## 3. From a0e2a0429db199198c2c5088b7b22a1d17a68868 Mon Sep 17 00:00:00 2001 From: VBoss Date: Sat, 26 Oct 2019 14:45:36 +0200 Subject: [PATCH 13/87] QuickStart guide - Configure Entity --- doc/00_quick_start.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/00_quick_start.md b/doc/00_quick_start.md index bf446a45..3aebe4c2 100644 --- a/doc/00_quick_start.md +++ b/doc/00_quick_start.md @@ -7,6 +7,8 @@ Use composer to install this library. composer require spameri/elastic ``` +--- + ## 2. Configure You need to set up few things first, before you can dive into ElasticSearch. @@ -55,6 +57,8 @@ Next step is to configure your first entity. This entity is for e-shop product. - Sixth line is reference to where is object with entity configuration - Seventh line is where you can configure your entity within this neon +--- + ## 3. From 6e4cab7a3f704579cb3afbf30c2f83e0f5cba5ed Mon Sep 17 00:00:00 2001 From: VBoss Date: Sat, 26 Oct 2019 15:36:55 +0200 Subject: [PATCH 14/87] Quick start --- doc/00_quick_start.md | 389 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 388 insertions(+), 1 deletion(-) diff --git a/doc/00_quick_start.md b/doc/00_quick_start.md index 3aebe4c2..258e3a7b 100644 --- a/doc/00_quick_start.md +++ b/doc/00_quick_start.md @@ -59,9 +59,396 @@ Next step is to configure your first entity. This entity is for e-shop product. --- -## 3. +## 3. Create entity class +```php +id; +} + + +public function entityVariables(): array +{ + return \get_object_vars($this); +} +``` + +### Factory +````php +class SimpleProductFactory implements \Spameri\Elastic\Factory\IEntityFactory +{ + + public function create(\Spameri\ElasticQuery\Response\Result\Hit $hit) : \Generator + { + yield new \App\ProductModule\Entity\SimpleProduct( + new \Spameri\Elastic\Entity\Property\ElasticId($hit->id()), + $hit->getValue('databaseId'), + $hit->getValue('name'), + $hit->getValue('content'), + $hit->getValue('alias'), + $hit->getValue('image'), + $hit->getValue('price'), + $hit->getValue('availability'), + $hit->getValue('tags'), + $hit->getValue('categories') + ); + } + +} +```` + +### CollectionFactory +````php +class SimpleProductCollectionFactory implements \Spameri\Elastic\Factory\ICollectionFactory +{ + + public function create( + \Spameri\Elastic\Model\IService $service + , array $elasticIds = [] + , \Spameri\Elastic\Entity\IElasticEntity ... $entityCollection + ) : \Spameri\Elastic\Entity\IElasticEntityCollection + { + return new \App\ProductModule\Entity\ProductCollection($service, $elasticIds, ... $entityCollection); + } + +} +```` + +## 4. Index Configuring +````php +class SimpleProductConfig implements \Spameri\Elastic\Settings\IndexConfigInterface +{ + public function __construct( + string $indexName + ) + { + $this->indexName = $indexName; + } +} +```` + +`public function provide(): \Spameri\ElasticQuery\Mapping\Settings` + +````php +$settings = new \Spameri\ElasticQuery\Mapping\Settings($this->indexName); +$czechDictionary = new \Spameri\ElasticQuery\Mapping\Analyzer\Custom\CzechDictionary(); +$settings->addAnalyzer($czechDictionary); + +$lowerCase = new \Spameri\ElasticQuery\Mapping\Analyzer\Custom\Lowercase(); +$settings->addAnalyzer($lowerCase); +```` + +````php +$settings->addMappingField( + new \Spameri\ElasticQuery\Mapping\Settings\Mapping\Field( + 'databaseId', + \Spameri\Elastic\Model\ValidateMapping\AllowedValues::TYPE_KEYWORD + ) +); +$settings->addMappingField( + new \Spameri\ElasticQuery\Mapping\Settings\Mapping\Field( + 'name', + \Spameri\Elastic\Model\ValidateMapping\AllowedValues::TYPE_TEXT, + $czechDictionary + ) +); +$settings->addMappingField( + new \Spameri\ElasticQuery\Mapping\Settings\Mapping\Field( + 'content', + \Spameri\Elastic\Model\ValidateMapping\AllowedValues::TYPE_TEXT, + $czechDictionary + ) +); +```` + + +````php +$settings->addMappingField( + new \Spameri\ElasticQuery\Mapping\Settings\Mapping\Field( + 'tags', + \Spameri\Elastic\Model\ValidateMapping\AllowedValues::TYPE_TEXT, + $lowerCase + ) +); +```` + + +## 5. Export data to ElasticSearch + +````php +class ExportToElastic extends \Spameri\Elastic\Import\Run +{ + + public function __construct( + string $logDir = 'log', + \Symfony\Component\Console\Output\ConsoleOutput $output, + \Spameri\Elastic\Import\Run\NullLoggerHandler $loggerHandler, + \Spameri\Elastic\Import\Lock\NullLock $lock, + \Spameri\Elastic\Import\RunHandler\NullHandler $runHandler, + + \App\ProductModule\Model\ExportToElastic\DataProvider $dataProvider, + \App\ProductModule\Model\ExportToElastic\PrepareImportData $prepareImportData, + \App\ProductModule\Model\ExportToElastic\DataImport $dataImport, + + \Spameri\Elastic\Import\AfterImport\NullAfterImport $afterImport + ) + { + parent::__construct($logDir, $output, $loggerHandler, $lock, $runHandler, $dataProvider, $prepareImportData, $dataImport, $afterImport); + } + +} +```` + + +````php +class DataProvider implements \Spameri\Elastic\Import\DataProviderInterface +{ + public function provide(\Spameri\Elastic\Import\Run\Options $options): \Generator + { + $query = $this->connection->select('*')->from('table'); + + while ($hasResults) { + $items = $query->fetchAll($offset, $limit); + + yield from $items; + + if ( ! \count($items)) { + $hasResults = FALSE; + + } else { + $offset += $limit; + } + } + } +} +```` +````php + +class PrepareImportData implements \Spameri\Elastic\Import\PrepareImportDataInterface +{ + + public function prepare($entityData): \Spameri\Elastic\Entity\AbstractImport + { + $imageSrc = '//via.placeholder.com/150x150'; + $elasticId = NULL; + $tags = []; + $categories = []; + return new \App\ProductModule\Entity\SimpleProduct( + $elasticId, + $entityData['id'], + $entityData['name'], + $entityData['content_description'], + $entityData['alias'], + $imageSrc, + $entityData['amount'], + $entityData['availability_id'] === 1 ? 'Skladem' : 'Nedostupné', + $tags, + $categories + ); + } + +} +```` + +````php +class DataImport implements \Spameri\Elastic\Import\DataImportInterface +{ + + /** + * @param \App\ProductModule\Entity\SimpleProduct $entity + */ + public function import( + \Spameri\Elastic\Entity\AbstractImport $entity + ): \Spameri\Elastic\Import\ResponseInterface + { + $id = $this->productService->insert($entity); + + return new \Spameri\Elastic\Import\Response\SimpleResponse( + $id, + $entity + ); + } + +} +```` + +````php +$options = new \Spameri\Elastic\Import\Run\Options(600); + +// Clear index +try { + $this->delete->execute($this->simpleProductConfig->provide()->indexName()); +} catch (\Spameri\Elastic\Exception\ElasticSearchException $exception) {} + +// Create index +$this->create->execute( + $this->simpleProductConfig->provide()->indexName(), + $this->simpleProductConfig->provide()->toArray() +); + +// Export +$this->exportToElastic->execute($options); +```` + +## 6. Presenter, Form, Template +````php +class SimpleProductListPresenter extends \App\Presenter\BasePresenter +{ + + public function renderDefault($queryString): void + { + $query = $this->buildQuery($queryString); + + try { + $products = $this->productService->getAllBy($query); + + } catch (\Spameri\Elastic\Exception\ElasticSearchException $exception) { + $products = []; + } + + $this->getTemplate()->add( + 'products', + $products + ); + $this->getTemplate()->add( + 'queryString', + $queryString + ); + } + +} +```` + +````php +public function createComponentSearchForm() :\Nette\Application\UI\Form +{ + $form = new \Nette\Application\UI\Form(); + $form->addText('queryString', 'query') + ->setAttribute('class', 'inp-text suggest') + ; + + $form->addSubmit('search', 'Search'); + + $form->onSuccess[] = function () use ($form) { + $this->redirect( + 301, + ':Product:SimpleProductList:default', + [ + 'queryString' => $form->getValues()->queryString, + ] + ); + }; + + return $form; +} +```` + +````php +{control searchForm} +

You have searched: {$queryString}

+ +
+