Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New index service #9

Open
wants to merge 17 commits into
base: master
Choose a base branch
from
Open

New index service #9

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
c450507
Added Index implementation
poratuk Aug 29, 2017
7dffddd
Rename indexes to index
poratuk Aug 29, 2017
0967355
Added test.
poratuk Aug 29, 2017
89f3bb8
Inidexing working. Tests passed
poratuk Aug 30, 2017
4fa206e
Changed Elasticsearch indexing to types.
poratuk Sep 1, 2017
c48e427
Some small changes on create/remove indexes
poratuk Sep 1, 2017
3ed1835
Fixing tests to index nodes on node creating...
poratuk Sep 4, 2017
3ad80c7
Removed Elasticserch adapter from microkernel
poratuk Sep 4, 2017
d71bb1a
Remove first errored test. Use test client for Erkasticsearch (not in…
poratuk Sep 6, 2017
905e620
Added logout text to the kernel boot.
poratuk Sep 8, 2017
8351a44
Merge pull request #8 from poratuk/kernel-boot-queue
esokullu Sep 9, 2017
88867d4
Added MySQL test, Removed var_dump from kernel
poratuk Sep 10, 2017
453ee92
Fixed index tests
poratuk Sep 10, 2017
782a5b2
excluding optional index tests
Sep 11, 2017
596cbf3
bug fixes, code cleanup
Sep 11, 2017
7422fe8
bugfix
esokullu Jan 5, 2018
3ebcbcb
.
esokullu Jan 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Changed Elasticsearch indexing to types. 
Allow search by classes.
  • Loading branch information
@poratuk
poratuk committed Sep 1, 2017
commit 4fa206ebd27db493d6ac0760bb0edb89ca919c74
123 changes: 66 additions & 57 deletions src/Pho/Kernel/Services/Index/Adapters/Elasticsearch.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,31 +43,9 @@ public function __construct(Kernel $kernel, array $params = [])
if ($this->client->indices()->exists($indexParams)) {
$this->client->indices()->delete($indexParams);
}

if ( ! $this->client->indices()->exists($indexParams)) {
$params = [
'index' => $this->dbname,
'body' => [
'mappings' => [
$this->tablename => [
'_source' => [
'enabled' => true
],
'properties' =>
[
'id' => ['type' => 'string'],
'attr' =>
[
'properties' =>
[
'k' => ['type' => 'string'],
'v' => ['type' => 'string']
]
]
]
]
]
]
];
$params = ['index' => $this->dbname];

$this->client->indices()->create($params);
}
Expand All @@ -83,10 +61,12 @@ public function addToIndex(string $id, array $params, array $classes = []): void
{
$body = ['attr' => [], 'classes' => $classes, 'id' => $id];
foreach ($params as $key => $value) {
$body['attr'][] = ['k' => $key, 'v' => $value];
$body['attr'][] = ['k' => $key, 'v' => (string)$value];
}
$class = $this->getTypeFromClass($classes);

$this->client->index($this->createQuery($id, $body));
$query = $this->createQuery($class, $id, $body);
$this->client->index($query);
}

/**
Expand All @@ -98,17 +78,19 @@ public function addToIndex(string $id, array $params, array $classes = []): void
public function editInIndex(string $id, array $params, array $classes = []): void
{
//If node not founded in index - add it.
if ( ! empty($this->searchById($id))) {
$this->removeById($id);
$founded = $this->searchById($id);
if ( ! empty($founded)) {
$type = $this->getTypeFromClass($founded['classes']);
$this->removeById($type, $id);
}

//Update document if node are exists
$body = ['attr' => [], 'classes' => $classes];
foreach ($params as $key => $value) {
$body['attr'][] = ['k' => $key, 'v' => $value];
$body['attr'][] = ['k' => $key, 'v' => (string)$value];
}

$params = $this->createQuery($id, $body);
$params = $this->createQuery($this->getTypeFromClass($classes), $id, $body);
$this->client->index($params);
}

Expand All @@ -120,72 +102,99 @@ public function editInIndex(string $id, array $params, array $classes = []): voi
public function searchById(string $id): array
{
try {
$results = $this->client->get($this->createQuery($id, false));
} catch (Elasticsearch\Common\Exceptions\TransportException $e)
{
$query = ['query' => ['match' => ['id' => $id]]];
$params = $this->createQuery(null, null, $query);

$results = $this->client->search($params);
} catch (Elasticsearch\Common\Exceptions\TransportException $e) {
return false;
} catch (Elasticsearch\Common\Exceptions\Missing404Exception $e) {
return false;
}
$founded = $this->remapReturn($results);
if (isset($founded[0])) {
return $founded[0];
} else {
return $founded;
}
return $this->remapReturn($results);
}

/**
* Search in indexing DB all attributes of entity by its ID
* @param string $id uuid string
* @return array array with keys id, key, value
*/
public function removeById(string $id): array
public function removeById(string $type, string $id): array
{
return $this->client->delete($this->createQuery($id, false));
$params = $this->createQuery($type, $id, false);
return $this->client->delete($params);
}

/**
* Search by some params
* @param string $id uuid string
* @return array array with keys id, key, value
*/
public function searchInIndex(string $value, string $key = "", array $classes = array()): array
public function searchInIndex(string $value, string $key = null, array $classes = array()): array
{
$query = ['query' => []];
$query['query']['match']['attr.v'] = $value;
if (!empty($key)) {
$query['query']['match']['attr.k'] = $key;
$query = ['query' => ['bool' => ['must' => []]]];
$query['query']['bool']['must'][]['match']['attr.v'] = $value;
if ( ! is_null($key)) {
$query['query']['bool']['must'][]['match']['attr.k'] = $key;
}

$params = $this->createQuery(implode(',', (array)$classes), null, $query);
$results = $this->client->search($params);
return $this->getIdsList($this->remapReturn($results));
}

public function getTypeFromClass($classes) {
$type = 'entity';
$class = false;

if (is_array($classes)) {
$class = array_shift($classes);
}
if (!empty($classes)) {
//$query['query']['match']['classes'] = $classes; //Search by classes not ready yet

if (is_string($classes)) {
$class = $classes;
}

$params = $this->createQuery(null, $query);
unset($params['type']);
$results = $this->client->search($params);
if ($class){
$type = substr($class, strrpos($class, '\\') + 1);
}

return $this->getIdsList($this->remapReturn($results));

return $type;
}

public function remapReturn(array $results)
{
$return = array();
if (isset($results['hits']) && isset($results['hits']['hits'])) {
foreach ($results['hits']['hits'] as $founded) {
$results = [
'id' => $founded['id'],
$a = [
'id' => $founded['_source']['id'],
'attributes' => [],
'classes' => $founded['_source']['classes'],
];
foreach ($founded['_source']['attr'] as $attribute) {
$results['attributes'][$attribute['k']] = $attribute['v'];
$a['attributes'][$attribute['k']] = $attribute['v'];
}
array_push($return, $a);
}
} else if (isset($results['_id']) && isset($results['_source'])) {
$return['id'] = $results['_source']['id'];
$return['classes'] = $results['_source']['classes'];
$return['type'] = $results['_type'];
foreach ($results['_source']['attr'] as $attribute) {
$results['attributes'][$attribute['k']] = $attribute['v'];
$return['attributes'][$attribute['k']] = $attribute['v'];
}
}
return $return;
}

public function getIdsList($founded)
public function getIdsList($founded): array
{
$ids = [];
foreach ($founded as $entitys) {
Expand All @@ -195,12 +204,12 @@ public function getIdsList($founded)
return $ids;
}

public function createQuery(string $id = null, $where = []): array
public function createQuery(string $type = null, string $id = null, $where = []): array
{
$query = [
'index' => $this->dbname,
'type' => $this->tablename,
];
$query = ['index' => $this->dbname];
if ($type) {
$query['type'] = $type;
}

if ($where !== false)
{
Expand Down
4 changes: 2 additions & 2 deletions src/Pho/Kernel/Services/Index/Index.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ public function index(EntityInterface $entity, bool $new=false):void
*
* @return array
*/
public function search(string $value, string $key = "", array $classes = array()): array
public function search(string $value, string $key = null, array $classes = array()): array
{
return $this->searchInIndex($value, $key, $classes);
return $this->searchInIndex($value, $key, $classes);
}

/**
Expand Down
108 changes: 94 additions & 14 deletions tests/Pho/Kernel/Services/Indexing/IndexingTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
use Pho\Kernel\TestCase;
use Elasticsearch\ClientBuilder;

class IndexTestsss extends TestCase
class IndexTest extends TestCase
{

protected $client;
Expand All @@ -41,10 +41,54 @@ public function testCreatedIndex()
$this->assertTrue($this->kernel->index()->client->indices()->exists(['index' => getenv('INDEX_DB')]));
}

public function testGetIndex()
public function testSearchIndex()
{
$output = $this->kernel->index()->search('Value');
$this->assertSame([], $output);
$index = $this->kernel->index();
$document =
[
"id" => "b31c8f9e-a476-4b5d-ad53-36d44c9ed615",
"attr" => [
[
"k" => "Password",
"v" => "e10adc3949ba59abbe56e057f20f883e"
],
[
"k" => "JoinTime",
"v" => "1504210623"
],
[
"k" => "Birthday",
"v" => "411436800"
],
[
"k" => "About",
"v" => ""
],
[
"k" => "Key",
"v" => "Value"
]
],
"classes" => [
"PhoNetworksAutogenerated\\User" => "PhoNetworksAutogenerated\\User",
"Pho\\Kernel\\Foundation\\AbstractActorDP" => "Pho\\Kernel\\Foundation\\AbstractActorDP",
"Pho\\Kernel\\Foundation\\AbstractActor" => "Pho\\Kernel\\Foundation\\AbstractActor",
"Pho\\Framework\\Actor" => "Pho\\Framework\\Actor",
"Pho\\Lib\\Graph\\Node" => "Pho\\Lib\\Graph\\Node"
],
];

$params = ['index' => getenv('INDEX_DB'), 'type' => 'User', "id" => "b31c8f9e-a476-4b5d-ad53-36d44c9ed615", 'body' => $document];
$response = $this->client->index($params);
sleep(1);
if (isset($response['created']) && $response['created']) {
$params = ['index' => getenv('INDEX_DB'), 'type' => 'User', "id" => "b31c8f9e-a476-4b5d-ad53-36d44c9ed615"];

$output = $index->search('Value');
$this->assertSame($output, ['b31c8f9e-a476-4b5d-ad53-36d44c9ed615']);
} else {
$this->markTestSkipped('Elasticsearch create document fail');
}
}

public function testAppendIndex()
Expand All @@ -55,12 +99,14 @@ public function testAppendIndex()

$index->index($node, true);

$return = $this->client->get(['index' => getenv('INDEX_DB'), 'type' => getenv('INDEX_TABLE'), 'id' => $node->id()->toString()]);
$this->assertArrayHasKey('_source', $return);
$this->assertArrayHasKey('attr', $return['_source']);
$this->assertArrayHasKey(4, $return['_source']['attr']);
$this->assertArrayHasKey('v', $return['_source']['attr'][4]);
$this->assertSame($node->attributes()->Key, $return['_source']['attr'][4]['v']);
sleep(1);
$params = ['index' => getenv('INDEX_DB'), 'body' => ['query' => ['match' => ['id' => $node->id()->toString()]]]];

$return = $this->client->search($params);
$this->assertNotEmpty($return['hits']['hits']);
$this->assertArrayHasKey(4, $return['hits']['hits'][0]['_source']['attr']);
$this->assertArrayHasKey('v', $return['hits']['hits'][0]['_source']['attr'][4]);
$this->assertSame($node->attributes()->Key, $return['hits']['hits'][0]['_source']['attr'][4]['v']);
}

public function testEditIndex()
Expand All @@ -70,17 +116,51 @@ public function testEditIndex()
$node->attributes()->Key = 'Value';

$index->index($node, true);

sleep(1);
$node->attributes()->Key = 'Some new value';
$index->index($node);
sleep(1);
$params = ['index' => getenv('INDEX_DB'), 'type' => $index->getTypeFromClass(get_class($node)), 'id' => $node->id()->toString()];

$return = $this->client->get(['index' => getenv('INDEX_DB'), 'type' => getenv('INDEX_TABLE'), 'id' => $node->id()->toString()]);
$return = $this->client->get($params);

$this->assertArrayHasKey('_source', $return);
$this->assertArrayHasKey('attr', $return['_source']);
$this->assertTrue($return['found']);
$this->assertArrayHasKey(4, $return['_source']['attr']);
$this->assertArrayHasKey('v', $return['_source']['attr'][4]);
$this->assertSame($node->attributes()->Key, $return['_source']['attr'][4]['v']);
}

public function testSearchByKeyAndValue()
{
$index = $this->kernel->index();
$node = new \PhoNetworksAutogenerated\User($this->kernel, $this->kernel->graph(), "123456");
$node->attributes()->Key = 'Value';

$index->index($node, true);

sleep(1);

$return = $index->search($node->attributes()->Key, 'Key');

$this->assertNotEmpty($return);
$this->assertSame($node->id()->toString(), $return[0]);
}

public function testSearchByKeyAndClass()
{
$index = $this->kernel->index();
$node = new \PhoNetworksAutogenerated\User($this->kernel, $this->kernel->graph(), "123456");
$node->attributes()->Key = 'Value';
$class = $index->getTypeFromClass(get_class($node));

$index->index($node, true);

sleep(1);

$return = $index->search($node->attributes()->Key, null, [$class]);

$this->assertNotEmpty($return);
$this->assertSame($node->id()->toString(), $return[0]);
}

}