Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[rule based autotagging] add attribute value store #17342

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Star Tree] [Search] Resolving Date histogram with metric aggregation using star-tree ([#16674](https://github.com/opensearch-project/OpenSearch/pull/16674))
- [Star Tree] [Search] Extensible design to support different query and field types ([#17137](https://github.com/opensearch-project/OpenSearch/pull/17137))
- Improve performace of NumericTermAggregation by avoiding unnecessary sorting([#17252](https://github.com/opensearch-project/OpenSearch/pull/17252))
- [Rule Based Auto-tagging] Add in-memory attribute value store ([#17342](https://github.com/opensearch-project/OpenSearch/pull/17342))

### Dependencies
- Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))
Expand Down
1 change: 1 addition & 0 deletions plugins/workload-management/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ opensearchplugin {
}

dependencies {
implementation 'org.apache.commons:commons-collections4:4.4'
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.plugin.wlm.rule;
/**
* This package holds constructs for the Rule's in-memory storage, processing and syncing the in-memory view
* with the index view
*/
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.plugin.wlm.rule.storage;

import java.util.Optional;

/**
* This interface provides apis to store Rule attribute values
*/
public interface AttributeValueStore {
/**
* Adds the value to the data structure
* @param key to be added
* @param value to be added
*/
void addValue(String key, String value);
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved

/**
* removes the key and associated value from the data structure
* @param key to be removed
*/
void removeValue(String key);

/**
* Returns the value associated with the key
* @param key in the data structure
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved
* @return
*/
Optional<String> getValue(String key);

/**
* Clears all the keys and their associated values from the structure
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved
*/
void clear();

/**
* It returns the number of values stored
* @return count of key,val pairs in the store
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved
*/
int size();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.plugin.wlm.rule.storage;

import org.apache.commons.collections4.trie.PatriciaTrie;

import java.util.Map;
import java.util.Optional;

/**
* This is a patricia trie based implementation of AttributeValueStore
*/
public class TrieBasedStore implements AttributeValueStore {
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved
PatriciaTrie<String> trie;
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved

public TrieBasedStore(PatriciaTrie<String> trie) {
this.trie = trie;
}

@Override
public void addValue(String key, String value) {
trie.put(key, value);
}

@Override
public void removeValue(String key) {
trie.remove(key);
}

@Override
public Optional<String> getValue(String key) {
/**
* Since we are inserting prefixes into the trie and searching for larger strings
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved
* It is important to find the largest matching prefix key in the trie efficiently
* Hence we can do binary search
*/
String longestMatchingPrefix = findLongestMatchingPrefix(key);
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved

/**
* Now there are following cases for this prefix
* 1. There is a Rule which has this prefix as one of the attribute values. In this case we should return the
* Rule's label otherwise send empty
*/
for (Map.Entry<String, String> possibleMatch :trie.prefixMap(longestMatchingPrefix).entrySet()) {
if (key.startsWith(possibleMatch.getKey())) {
return Optional.of(possibleMatch.getValue());
}
}

return Optional.empty();
}

private String findLongestMatchingPrefix(String key) {
int low = 0;
int high = key.length()-1;

while (low < high) {
int mid = low + (high - low + 1) / 2;
String possibleMatchingPrefix = key.substring(0, mid);

if (!trie.prefixMap(possibleMatchingPrefix).isEmpty()) {
kaushalmahi12 marked this conversation as resolved.
Show resolved Hide resolved
low = mid;
} else {
high = mid-1;
}
}

return key.substring(0, low);
}

@Override
public void clear() {
trie.clear();
}

@Override
public int size() {
return trie.size();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.plugin.wlm.rule.storage;
/**
* This package contains interfaces and implementations for in memory rule storage mechanisms
*/
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.plugin.wlm.rule.storage;

import org.apache.commons.collections4.trie.PatriciaTrie;
import org.opensearch.test.OpenSearchTestCase;


public class AttributeValueStoreTests extends OpenSearchTestCase {

AttributeValueStore subjectUnderTest;

public void setUp() throws Exception {
super.setUp();
subjectUnderTest = new TrieBasedStore(new PatriciaTrie<>());
}

public void testAddValue() {
subjectUnderTest.addValue("foo", "bar");
assertEquals("bar", subjectUnderTest.getValue("foo").get());
}

public void testRemoveValue() {
subjectUnderTest.addValue("foo", "bar");
subjectUnderTest.removeValue("foo");
assertEquals(0, subjectUnderTest.size());
}

public void tesGetValue() {
subjectUnderTest.addValue("foo", "bar");
assertEquals("bar", subjectUnderTest.getValue("foo").get());
}

public void testGetValueWhenNoProperPrefixIsPresent() {
subjectUnderTest.addValue("foo", "bar");
subjectUnderTest.addValue("foodip", "sing");
assertTrue(subjectUnderTest.getValue("foxtail").isEmpty());
subjectUnderTest.addValue("fox", "lucy");

assertFalse(subjectUnderTest.getValue("foxtail").isEmpty());
}


public void testClear() {
subjectUnderTest.addValue("foo", "bar");
subjectUnderTest.clear();
assertEquals(0, subjectUnderTest.size());
}
}
Loading