#3016 New expand PPL command

core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java

@@ -8,6 +8,7 @@
 package org.opensearch.sql.analysis;
 
 import java.util.List;
+import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.datasource.DataSourceService;
 
 public class DataSourceSchemaIdentifierNameResolver {
@@ -21,8 +22,6 @@ public class DataSourceSchemaIdentifierNameResolver {
   private final String identifierName;
   private final DataSourceService dataSourceService;
 
-  private static final String DOT = ".";
-
   /**
    * Data model for capturing dataSourceName, schema and identifier from fully qualifiedName. In the
    * current state, it is used to capture DataSourceSchemaTable name and DataSourceSchemaFunction in
@@ -35,7 +34,7 @@ public DataSourceSchemaIdentifierNameResolver(
       DataSourceService dataSourceService, List<String> parts) {
     this.dataSourceService = dataSourceService;
     List<String> remainingParts = captureSchemaName(captureDataSourceName(parts));
-    identifierName = String.join(DOT, remainingParts);
+    identifierName = ExprValueUtils.joinQualifiedName(remainingParts);
   }
 
   public String getIdentifierName() {

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -44,9 +44,11 @@
 import org.opensearch.sql.ast.tree.CloseCursor;
 import org.opensearch.sql.ast.tree.Dedupe;
 import org.opensearch.sql.ast.tree.Eval;
+import org.opensearch.sql.ast.tree.Expand;
 import org.opensearch.sql.ast.tree.FetchCursor;
 import org.opensearch.sql.ast.tree.FillNull;
 import org.opensearch.sql.ast.tree.Filter;
+import org.opensearch.sql.ast.tree.Flatten;
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Kmeans;
 import org.opensearch.sql.ast.tree.Limit;
@@ -103,10 +105,18 @@ public T visitRelationSubquery(RelationSubquery node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitExpand(Expand node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitTableFunction(TableFunction node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitFlatten(Flatten node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitFilter(Filter node, C context) {
     return visitChildren(node, context);
   }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -49,8 +49,10 @@
 import org.opensearch.sql.ast.tree.Aggregation;
 import org.opensearch.sql.ast.tree.Dedupe;
 import org.opensearch.sql.ast.tree.Eval;
+import org.opensearch.sql.ast.tree.Expand;
 import org.opensearch.sql.ast.tree.FillNull;
 import org.opensearch.sql.ast.tree.Filter;
+import org.opensearch.sql.ast.tree.Flatten;
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.Parse;
@@ -104,6 +106,14 @@ public static Eval eval(UnresolvedPlan input, Let... projectList) {
     return new Eval(Arrays.asList(projectList)).attach(input);
   }
 
+  public Expand expand(UnresolvedPlan input, Field field) {
+    return new Expand(field).attach(input);
+  }
+
+  public Flatten flatten(UnresolvedPlan input, Field field) {
+    return new Flatten(field).attach(input);
+  }
+
   public static UnresolvedPlan projectWithArg(
       UnresolvedPlan input, List<Argument> argList, UnresolvedExpression... projectList) {
     return new Project(Arrays.asList(projectList), argList).attach(input);

core/src/main/java/org/opensearch/sql/ast/tree/Eval.java

@@ -10,14 +10,12 @@
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
-import lombok.Setter;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.expression.Let;
 
 /** AST node represent Eval operation. */
 @Getter
-@Setter
 @ToString
 @EqualsAndHashCode(callSuper = false)
 @RequiredArgsConstructor

core/src/main/java/org/opensearch/sql/ast/tree/Expand.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Field;
+
+/** AST node representing an {@code expand <field>} operation. */
+@Getter
+@ToString
+@RequiredArgsConstructor
+public class Expand extends UnresolvedPlan {
+
+  private UnresolvedPlan child;
+  @Getter private final Field field;
+
+  @Override
+  public Expand attach(UnresolvedPlan child) {
+    this.child = child;
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return ImmutableList.of(child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitExpand(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/tree/Flatten.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Field;
+
+/** AST node representing a {@code flatten <field>} operation. */
+@ToString
+@RequiredArgsConstructor
+public class Flatten extends UnresolvedPlan {
+
+  private UnresolvedPlan child;
+  @Getter private final Field field;
+
+  @Override
+  public Flatten attach(UnresolvedPlan child) {
+    this.child = child;
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return ImmutableList.of(child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitFlatten(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/tree/Relation.java

@@ -67,7 +67,7 @@ public String getAlias() {
   }
 
   /**
-   * Get Qualified name preservs parts of the user given identifiers. This can later be utilized to
+   * Get Qualified name preserves parts of the user given identifiers. This can later be utilized to
    * determine DataSource,Schema and Table Name during Analyzer stage. So Passing QualifiedName
    * directly to Analyzer Stage.
    *

core/src/main/java/org/opensearch/sql/data/model/ExprIntervalValue.java

@@ -7,6 +7,7 @@
 
 import java.time.temporal.TemporalAmount;
 import java.time.temporal.TemporalUnit;
+import java.util.Objects;
 import lombok.RequiredArgsConstructor;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.data.type.ExprType;
@@ -56,4 +57,9 @@ public TemporalUnit unit() {
         .findAny()
         .orElse(interval.getUnits().get(0));
   }
+
+  @Override
+  public int hashCode() {
+    return Objects.hashCode(interval);
+  }
 }

core/src/main/java/org/opensearch/sql/data/model/ExprIpValue.java

@@ -6,6 +6,7 @@
 package org.opensearch.sql.data.model;
 
 import inet.ipaddr.IPAddress;
+import java.util.Objects;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.data.type.ExprType;
 import org.opensearch.sql.utils.IPUtils;
@@ -47,4 +48,9 @@ public String toString() {
   public IPAddress ipValue() {
     return value;
   }
+
+  @Override
+  public int hashCode() {
+    return Objects.hashCode(value);
+  }
 }

core/src/main/java/org/opensearch/sql/data/model/ExprTupleValue.java

@@ -5,7 +5,6 @@
 
 package org.opensearch.sql.data.model;
 
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -69,23 +68,20 @@ public ExprValue keyValue(String key) {
    *
    * @return true for equal, otherwise false.
    */
-  public boolean equal(ExprValue o) {
-    if (!(o instanceof ExprTupleValue)) {
+  @Override
+  public boolean equal(ExprValue other) {
+
+    if (!(other instanceof ExprTupleValue)) {
       return false;
-    } else {
-      ExprTupleValue other = (ExprTupleValue) o;
-      Iterator<Entry<String, ExprValue>> thisIterator = this.valueMap.entrySet().iterator();
-      Iterator<Entry<String, ExprValue>> otherIterator = other.valueMap.entrySet().iterator();
-      while (thisIterator.hasNext() && otherIterator.hasNext()) {
-        Entry<String, ExprValue> thisEntry = thisIterator.next();
-        Entry<String, ExprValue> otherEntry = otherIterator.next();
-        if (!(thisEntry.getKey().equals(otherEntry.getKey())
-            && thisEntry.getValue().equals(otherEntry.getValue()))) {
-          return false;
-        }
-      }
-      return !(thisIterator.hasNext() || otherIterator.hasNext());
     }
+
+    /**
+     * {@link Map#equals} returns true if the two maps' entry sets are equal, and works properly
+     * across all implementation of the {@link Map} interface. See {@link
+     * https://docs.oracle.com/javase/8/docs/api/java/util/Map.html#equals-java.lang.Object-} for
+     * more details.
+     */
+    return valueMap.equals(other.tupleValue());
   }
 
   /** Only compare the size of the map. */

core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java

@@ -13,21 +13,32 @@
 import java.time.ZoneOffset;
 import java.time.temporal.TemporalAmount;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 import lombok.experimental.UtilityClass;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.exception.ExpressionEvaluationException;
 
 /** The definition of {@link ExprValue} factory. */
 @UtilityClass
 public class ExprValueUtils {
+
+  // Literal constants
   public static final ExprValue LITERAL_TRUE = ExprBooleanValue.of(true);
   public static final ExprValue LITERAL_FALSE = ExprBooleanValue.of(false);
   public static final ExprValue LITERAL_NULL = ExprNullValue.of();
   public static final ExprValue LITERAL_MISSING = ExprMissingValue.of();
 
+  /** Qualified name separator string */
+  private final String QUALIFIED_NAME_SEPARATOR = ".";
+
+  /** Pattern that matches the qualified name separator string */
+  private final Pattern QUALIFIED_NAME_SEPARATOR_PATTERN =
+      Pattern.compile(QUALIFIED_NAME_SEPARATOR, Pattern.LITERAL);
+
   public static ExprValue booleanValue(Boolean value) {
     return value ? LITERAL_TRUE : LITERAL_FALSE;
   }
@@ -200,4 +211,14 @@ public static IPAddress getIpValue(ExprValue exprValue) {
   public static Boolean getBooleanValue(ExprValue exprValue) {
     return exprValue.booleanValue();
   }
+
+  /** Splits the given qualified name into components and returns the result. */
+  public List<String> splitQualifiedName(String qualifiedName) {
+    return Arrays.asList(QUALIFIED_NAME_SEPARATOR_PATTERN.split(qualifiedName, -1));
+  }
+
+  /** Joins the given components into a qualified name and returns the result. */
+  public String joinQualifiedName(List<String> components) {
+    return String.join(QUALIFIED_NAME_SEPARATOR, components);
+  }
 }

core/src/main/java/org/opensearch/sql/executor/Explain.java

@@ -22,7 +22,9 @@
 import org.opensearch.sql.planner.physical.AggregationOperator;
 import org.opensearch.sql.planner.physical.DedupeOperator;
 import org.opensearch.sql.planner.physical.EvalOperator;
+import org.opensearch.sql.planner.physical.ExpandOperator;
 import org.opensearch.sql.planner.physical.FilterOperator;
+import org.opensearch.sql.planner.physical.FlattenOperator;
 import org.opensearch.sql.planner.physical.LimitOperator;
 import org.opensearch.sql.planner.physical.NestedOperator;
 import org.opensearch.sql.planner.physical.PhysicalPlan;
@@ -160,6 +162,23 @@ public ExplainResponseNode visitEval(EvalOperator node, Object context) {
                 ImmutableMap.of("expressions", convertPairListToMap(node.getExpressionList()))));
   }
 
+  @Override
+  public ExplainResponseNode visitExpand(ExpandOperator node, Object context) {
+    return explain(
+        node,
+        context,
+        explainNode -> explainNode.setDescription(ImmutableMap.of("expandField", node.getField())));
+  }
+
+  @Override
+  public ExplainResponseNode visitFlatten(FlattenOperator node, Object context) {
+    return explain(
+        node,
+        context,
+        explainNode ->
+            explainNode.setDescription(ImmutableMap.of("flattenField", node.getField())));
+  }
+
   @Override
   public ExplainResponseNode visitDedupe(DedupeOperator node, Object context) {
     return explain(

core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java

@@ -5,15 +5,14 @@
 
 package org.opensearch.sql.expression;
 
-import static org.opensearch.sql.utils.ExpressionUtils.PATH_SEP;
-
 import java.util.Arrays;
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import org.opensearch.sql.data.model.ExprTupleValue;
 import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.data.type.ExprType;
 import org.opensearch.sql.expression.env.Environment;
@@ -106,7 +105,7 @@ public ExprValue resolve(ExprTupleValue value) {
   }
 
   private ExprValue resolve(ExprValue value, List<String> paths) {
-    ExprValue wholePathValue = value.keyValue(String.join(PATH_SEP, paths));
+    ExprValue wholePathValue = value.keyValue(ExprValueUtils.joinQualifiedName(paths));
     // For array types only first index currently supported.
     if (value.type().equals(ExprCoreType.ARRAY)) {
       wholePathValue = value.collectionValue().get(0).keyValue(paths.get(0));