diff --git a/Cargo.lock b/Cargo.lock index bb2d74f28b32..42857f1e18f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4692,7 +4692,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=e2fd89fce1fe9ea0c36c85bcf447ce4bb4a84af3#e2fd89fce1fe9ea0c36c85bcf447ce4bb4a84af3" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=fc09a5696608d2a0aa718cc835d5cb9c4e8e9387#fc09a5696608d2a0aa718cc835d5cb9c4e8e9387" dependencies = [ "prost 0.13.3", "serde", diff --git a/Cargo.toml b/Cargo.toml index b4543e5e4d66..5b9f893f1980 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,7 +129,7 @@ etcd-client = "0.14" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "e2fd89fce1fe9ea0c36c85bcf447ce4bb4a84af3" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "fc09a5696608d2a0aa718cc835d5cb9c4e8e9387" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/api/src/v1/column_def.rs b/src/api/src/v1/column_def.rs index a1d209c0e74f..214418bd44b8 100644 --- a/src/api/src/v1/column_def.rs +++ b/src/api/src/v1/column_def.rs @@ -15,10 +15,10 @@ use std::collections::HashMap; use datatypes::schema::{ - ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextOptions, COMMENT_KEY, - FULLTEXT_KEY, INVERTED_INDEX_KEY, SKIPPING_INDEX_KEY, + ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextOptions, SkippingIndexType, + COMMENT_KEY, FULLTEXT_KEY, INVERTED_INDEX_KEY, SKIPPING_INDEX_KEY, }; -use greptime_proto::v1::Analyzer; +use greptime_proto::v1::{Analyzer, SkippingIndexType as PbSkippingIndexType}; use snafu::ResultExt; use crate::error::{self, Result}; @@ -121,6 +121,13 @@ pub fn as_fulltext_option(analyzer: Analyzer) -> FulltextAnalyzer { } } +/// Tries to construct a `SkippingIndexType` from the given skipping index type. +pub fn as_skipping_index_type(skipping_index_type: PbSkippingIndexType) -> SkippingIndexType { + match skipping_index_type { + PbSkippingIndexType::BloomFilter => SkippingIndexType::BloomFilter, + } +} + #[cfg(test)] mod tests { diff --git a/src/common/grpc-expr/src/alter.rs b/src/common/grpc-expr/src/alter.rs index 78ceff0d2f35..85f8e51d37fb 100644 --- a/src/common/grpc-expr/src/alter.rs +++ b/src/common/grpc-expr/src/alter.rs @@ -15,13 +15,14 @@ use api::helper::ColumnDataTypeWrapper; use api::v1::add_column_location::LocationType; use api::v1::alter_table_expr::Kind; -use api::v1::column_def::as_fulltext_option; +use api::v1::column_def::{as_fulltext_option, as_skipping_index_type}; use api::v1::{ column_def, AddColumnLocation as Location, AlterTableExpr, Analyzer, CreateTableExpr, DropColumns, ModifyColumnTypes, RenameTable, SemanticType, + SkippingIndexType as PbSkippingIndexType, }; use common_query::AddColumnLocation; -use datatypes::schema::{ColumnSchema, FulltextOptions, RawSchema}; +use datatypes::schema::{ColumnSchema, FulltextOptions, RawSchema, SkippingIndexOptions}; use snafu::{ensure, OptionExt, ResultExt}; use store_api::region_request::{SetRegionOption, UnsetRegionOption}; use table::metadata::TableId; @@ -31,7 +32,8 @@ use table::requests::{ }; use crate::error::{ - InvalidColumnDefSnafu, InvalidSetFulltextOptionRequestSnafu, InvalidSetTableOptionRequestSnafu, + InvalidColumnDefSnafu, InvalidSetFulltextOptionRequestSnafu, + InvalidSetSkippingIndexOptionRequestSnafu, InvalidSetTableOptionRequestSnafu, InvalidUnsetTableOptionRequestSnafu, MissingAlterIndexOptionSnafu, MissingFieldSnafu, MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu, }; @@ -137,6 +139,18 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterTableExpr) -> Result< column_name: i.column_name, }, }, + api::v1::set_index::Options::Skipping(s) => AlterKind::SetIndex { + options: SetIndexOptions::Skipping { + column_name: s.column_name, + options: SkippingIndexOptions { + granularity: s.granularity as u32, + index_type: as_skipping_index_type( + PbSkippingIndexType::try_from(s.skipping_index_type) + .context(InvalidSetSkippingIndexOptionRequestSnafu)?, + ), + }, + }, + }, }, None => return MissingAlterIndexOptionSnafu.fail(), }, @@ -152,6 +166,11 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterTableExpr) -> Result< column_name: i.column_name, }, }, + api::v1::unset_index::Options::Skipping(s) => AlterKind::UnsetIndex { + options: UnsetIndexOptions::Skipping { + column_name: s.column_name, + }, + }, }, None => return MissingAlterIndexOptionSnafu.fail(), }, diff --git a/src/common/grpc-expr/src/error.rs b/src/common/grpc-expr/src/error.rs index 7eca56bccd41..5092181f7682 100644 --- a/src/common/grpc-expr/src/error.rs +++ b/src/common/grpc-expr/src/error.rs @@ -140,6 +140,14 @@ pub enum Error { error: prost::UnknownEnumValue, }, + #[snafu(display("Invalid set skipping index option request"))] + InvalidSetSkippingIndexOptionRequest { + #[snafu(implicit)] + location: Location, + #[snafu(source)] + error: prost::UnknownEnumValue, + }, + #[snafu(display("Missing alter index options"))] MissingAlterIndexOption { #[snafu(implicit)] @@ -171,6 +179,7 @@ impl ErrorExt for Error { Error::InvalidSetTableOptionRequest { .. } | Error::InvalidUnsetTableOptionRequest { .. } | Error::InvalidSetFulltextOptionRequest { .. } + | Error::InvalidSetSkippingIndexOptionRequest { .. } | Error::MissingAlterIndexOption { .. } => StatusCode::InvalidArguments, } } diff --git a/src/datatypes/src/schema/column_schema.rs b/src/datatypes/src/schema/column_schema.rs index c403600ad166..6547886dfbbe 100644 --- a/src/datatypes/src/schema/column_schema.rs +++ b/src/datatypes/src/schema/column_schema.rs @@ -380,6 +380,11 @@ impl ColumnSchema { ); Ok(()) } + + pub fn unset_skipping_options(&mut self) -> Result<()> { + self.metadata.remove(SKIPPING_INDEX_KEY); + Ok(()) + } } /// Column extended type set in column schema's metadata. diff --git a/src/operator/src/expr_helper.rs b/src/operator/src/expr_helper.rs index bb22ae6fdf29..3c90f4d27571 100644 --- a/src/operator/src/expr_helper.rs +++ b/src/operator/src/expr_helper.rs @@ -22,14 +22,15 @@ use api::v1::{ set_index, unset_index, AddColumn, AddColumns, AlterDatabaseExpr, AlterTableExpr, Analyzer, ColumnDataType, ColumnDataTypeExtension, CreateFlowExpr, CreateTableExpr, CreateViewExpr, DropColumn, DropColumns, ExpireAfter, ModifyColumnType, ModifyColumnTypes, RenameTable, - SemanticType, SetDatabaseOptions, SetFulltext, SetIndex, SetInverted, SetTableOptions, - TableName, UnsetDatabaseOptions, UnsetFulltext, UnsetIndex, UnsetInverted, UnsetTableOptions, + SemanticType, SetDatabaseOptions, SetFulltext, SetIndex, SetInverted, SetSkipping, + SetTableOptions, SkippingIndexType as PbSkippingIndexType, TableName, UnsetDatabaseOptions, + UnsetFulltext, UnsetIndex, UnsetInverted, UnsetSkipping, UnsetTableOptions, }; use common_error::ext::BoxedError; use common_grpc_expr::util::ColumnExpr; use common_time::Timezone; use datafusion::sql::planner::object_name_to_table_reference; -use datatypes::schema::{ColumnSchema, FulltextAnalyzer, Schema, COMMENT_KEY}; +use datatypes::schema::{ColumnSchema, FulltextAnalyzer, Schema, SkippingIndexType, COMMENT_KEY}; use file_engine::FileOptions; use query::sql::{ check_file_to_table_schema_compatibility, file_column_schemas_to_table, @@ -587,6 +588,19 @@ pub(crate) fn to_alter_table_expr( column_name: column_name.value, })), }, + sql::statements::alter::SetIndexOperation::Skipping { + column_name, + options, + } => SetIndex { + options: Some(set_index::Options::Skipping(SetSkipping { + column_name: column_name.value, + enable: true, + granularity: options.granularity as u64, + skipping_index_type: match options.index_type { + SkippingIndexType::BloomFilter => PbSkippingIndexType::BloomFilter.into(), + }, + })), + }, }), AlterTableOperation::UnsetIndex { options } => AlterTableKind::UnsetIndex(match options { sql::statements::alter::UnsetIndexOperation::Fulltext { column_name } => UnsetIndex { @@ -599,6 +613,11 @@ pub(crate) fn to_alter_table_expr( column_name: column_name.value, })), }, + sql::statements::alter::UnsetIndexOperation::Skipping { column_name } => UnsetIndex { + options: Some(unset_index::Options::Skipping(UnsetSkipping { + column_name: column_name.value, + })), + }, }), }; diff --git a/src/sql/src/parsers/alter_parser.rs b/src/sql/src/parsers/alter_parser.rs index 87636efed6d9..0411292dae13 100644 --- a/src/sql/src/parsers/alter_parser.rs +++ b/src/sql/src/parsers/alter_parser.rs @@ -25,7 +25,9 @@ use sqlparser::tokenizer::{Token, TokenWithLocation}; use crate::error::{self, InvalidColumnOptionSnafu, Result, SetFulltextOptionSnafu}; use crate::parser::ParserContext; use crate::parsers::create_parser::INVERTED; -use crate::parsers::utils::validate_column_fulltext_create_option; +use crate::parsers::utils::{ + validate_column_fulltext_create_option, validate_column_skipping_index_create_option, +}; use crate::statements::alter::{ AddColumn, AlterDatabase, AlterDatabaseOperation, AlterTable, AlterTableOperation, KeyValueOption, SetIndexOperation, UnsetIndexOperation, @@ -241,9 +243,14 @@ impl ParserContext<'_> { TokenWithLocation { token: Token::Word(w), .. - } if w.keyword == Keyword::FULLTEXT => Ok(AlterTableOperation::UnsetIndex { - options: UnsetIndexOperation::Fulltext { column_name }, - }), + } if w.keyword == Keyword::FULLTEXT => { + self.parser + .expect_keyword(Keyword::INDEX) + .context(error::SyntaxSnafu)?; + Ok(AlterTableOperation::UnsetIndex { + options: UnsetIndexOperation::Fulltext { column_name }, + }) + } TokenWithLocation { token: Token::Word(w), @@ -256,8 +263,24 @@ impl ParserContext<'_> { options: UnsetIndexOperation::Inverted { column_name }, }) } + + TokenWithLocation { + token: Token::Word(w), + .. + } if w.value.eq_ignore_ascii_case("SKIPPING") => { + self.parser + .expect_keyword(Keyword::INDEX) + .context(error::SyntaxSnafu)?; + Ok(AlterTableOperation::UnsetIndex { + options: UnsetIndexOperation::Skipping { column_name }, + }) + } _ => self.expected( - format!("{:?} OR INVERTED INDEX", Keyword::FULLTEXT).as_str(), + format!( + "{:?} OR INVERTED INDEX OR SKIPPING INDEX", + Keyword::FULLTEXT + ) + .as_str(), self.parser.peek_token(), ), } @@ -268,7 +291,12 @@ impl ParserContext<'_> { TokenWithLocation { token: Token::Word(w), .. - } if w.keyword == Keyword::FULLTEXT => self.parse_alter_column_fulltext(column_name), + } if w.keyword == Keyword::FULLTEXT => { + self.parser + .expect_keyword(Keyword::INDEX) + .context(error::SyntaxSnafu)?; + self.parse_alter_column_fulltext(column_name) + } TokenWithLocation { token: Token::Word(w), @@ -281,8 +309,18 @@ impl ParserContext<'_> { options: SetIndexOperation::Inverted { column_name }, }) } + + TokenWithLocation { + token: Token::Word(w), + .. + } if w.value.eq_ignore_ascii_case("SKIPPING") => { + self.parser + .expect_keyword(Keyword::INDEX) + .context(error::SyntaxSnafu)?; + self.parse_alter_column_skipping(column_name) + } _ => self.expected( - format!("{:?} OR INVERTED INDEX", Keyword::FULLTEXT).as_str(), + format!("{:?} OR INVERTED OR SKIPPING INDEX", Keyword::FULLTEXT).as_str(), self.parser.peek_token(), ), } @@ -319,6 +357,35 @@ impl ParserContext<'_> { }, }) } + + fn parse_alter_column_skipping(&mut self, column_name: Ident) -> Result { + let options = self + .parser + .parse_options(Keyword::WITH) + .context(error::SyntaxSnafu)? + .into_iter() + .map(parse_option_string) + .collect::>>()?; + + for key in options.keys() { + ensure!( + validate_column_skipping_index_create_option(key), + InvalidColumnOptionSnafu { + name: column_name.to_string(), + msg: format!("invalid SKIPPING INDEX option: {key}"), + } + ); + } + + Ok(AlterTableOperation::SetIndex { + options: SetIndexOperation::Skipping { + column_name, + options: options + .try_into() + .context(error::SetSkippingIndexOptionSnafu)?, + }, + }) + } } /// Parses a string literal and an optional string literal value. @@ -891,7 +958,7 @@ mod tests { #[test] fn test_parse_alter_column_fulltext() { - let sql = "ALTER TABLE test_table MODIFY COLUMN a SET FULLTEXT WITH(analyzer='English',case_sensitive='false')"; + let sql = "ALTER TABLE test_table MODIFY COLUMN a SET FULLTEXT INDEX WITH(analyzer='English',case_sensitive='false')"; let mut result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) .unwrap(); @@ -928,7 +995,7 @@ mod tests { _ => unreachable!(), } - let sql = "ALTER TABLE test_table MODIFY COLUMN a UNSET FULLTEXT"; + let sql = "ALTER TABLE test_table MODIFY COLUMN a UNSET FULLTEXT INDEX"; let mut result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) .unwrap(); @@ -955,7 +1022,8 @@ mod tests { _ => unreachable!(), } - let invalid_sql = "ALTER TABLE test_table MODIFY COLUMN a SET FULLTEXT WITH('abcd'='true')"; + let invalid_sql = + "ALTER TABLE test_table MODIFY COLUMN a SET FULLTEXT INDEX WITH('abcd'='true')"; let result = ParserContext::create_with_dialect( invalid_sql, &GreptimeDbDialect {}, diff --git a/src/sql/src/statements/alter.rs b/src/sql/src/statements/alter.rs index f807b4f07efa..4f271efc4096 100644 --- a/src/sql/src/statements/alter.rs +++ b/src/sql/src/statements/alter.rs @@ -16,7 +16,7 @@ use std::fmt::{Debug, Display}; use api::v1; use common_query::AddColumnLocation; -use datatypes::schema::FulltextOptions; +use datatypes::schema::{FulltextOptions, SkippingIndexOptions}; use itertools::Itertools; use serde::Serialize; use sqlparser::ast::{ColumnDef, DataType, Ident, ObjectName, TableConstraint}; @@ -96,22 +96,28 @@ pub enum AlterTableOperation { #[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut, Serialize)] pub enum SetIndexOperation { - /// `MODIFY COLUMN SET FULLTEXT [WITH ]` + /// `MODIFY COLUMN SET FULLTEXT INDEX [WITH ]` Fulltext { column_name: Ident, options: FulltextOptions, }, /// `MODIFY COLUMN SET INVERTED INDEX` Inverted { column_name: Ident }, + /// `MODIFY COLUMN SET SKIPPING INDEX` + Skipping { + column_name: Ident, + options: SkippingIndexOptions, + }, } #[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut, Serialize)] pub enum UnsetIndexOperation { - /// `MODIFY COLUMN UNSET FULLTEXT` + /// `MODIFY COLUMN UNSET FULLTEXT INDEX` Fulltext { column_name: Ident }, - /// `MODIFY COLUMN UNSET INVERTED INDEX` Inverted { column_name: Ident }, + /// `MODIFY COLUMN UNSET SKIPPING INDEX` + Skipping { column_name: Ident }, } #[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut, Serialize)] @@ -175,19 +181,28 @@ impl Display for AlterTableOperation { column_name, options, } => { - write!(f, "MODIFY COLUMN {column_name} SET FULLTEXT WITH(analyzer={0}, case_sensitive={1})", options.analyzer, options.case_sensitive) + write!(f, "MODIFY COLUMN {column_name} SET FULLTEXT INDEX WITH(analyzer={0}, case_sensitive={1})", options.analyzer, options.case_sensitive) } SetIndexOperation::Inverted { column_name } => { write!(f, "MODIFY COLUMN {column_name} SET INVERTED INDEX") } + SetIndexOperation::Skipping { + column_name, + options, + } => { + write!(f, "MODIFY COLUMN {column_name} SET SKIPPING INDEX WITH(granularity={0}, index_type={1})", options.granularity, options.index_type) + } }, AlterTableOperation::UnsetIndex { options } => match options { UnsetIndexOperation::Fulltext { column_name } => { - write!(f, "MODIFY COLUMN {column_name} UNSET FULLTEXT") + write!(f, "MODIFY COLUMN {column_name} UNSET FULLTEXT INDEX") } UnsetIndexOperation::Inverted { column_name } => { write!(f, "MODIFY COLUMN {column_name} UNSET INVERTED INDEX") } + UnsetIndexOperation::Skipping { column_name } => { + write!(f, "MODIFY COLUMN {column_name} UNSET SKIPPING INDEX") + } }, } } @@ -410,7 +425,7 @@ ALTER TABLE monitor RENAME monitor_new"#, } } - let sql = "ALTER TABLE monitor MODIFY COLUMN a SET FULLTEXT WITH(analyzer='English',case_sensitive='false')"; + let sql = "ALTER TABLE monitor MODIFY COLUMN a SET FULLTEXT INDEX WITH(analyzer='English',case_sensitive='false')"; let stmts = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) .unwrap(); @@ -422,7 +437,7 @@ ALTER TABLE monitor RENAME monitor_new"#, let new_sql = format!("\n{}", set); assert_eq!( r#" -ALTER TABLE monitor MODIFY COLUMN a SET FULLTEXT WITH(analyzer=English, case_sensitive=false)"#, +ALTER TABLE monitor MODIFY COLUMN a SET FULLTEXT INDEX WITH(analyzer=English, case_sensitive=false)"#, &new_sql ); } @@ -431,7 +446,7 @@ ALTER TABLE monitor MODIFY COLUMN a SET FULLTEXT WITH(analyzer=English, case_sen } } - let sql = "ALTER TABLE monitor MODIFY COLUMN a UNSET FULLTEXT"; + let sql = "ALTER TABLE monitor MODIFY COLUMN a UNSET FULLTEXT INDEX"; let stmts = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) .unwrap(); @@ -443,7 +458,7 @@ ALTER TABLE monitor MODIFY COLUMN a SET FULLTEXT WITH(analyzer=English, case_sen let new_sql = format!("\n{}", set); assert_eq!( r#" -ALTER TABLE monitor MODIFY COLUMN a UNSET FULLTEXT"#, +ALTER TABLE monitor MODIFY COLUMN a UNSET FULLTEXT INDEX"#, &new_sql ); } diff --git a/src/store-api/src/metadata.rs b/src/store-api/src/metadata.rs index b07e1bdb60f7..e776c63bc948 100644 --- a/src/store-api/src/metadata.rs +++ b/src/store-api/src/metadata.rs @@ -28,7 +28,7 @@ use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; use datatypes::arrow::datatypes::FieldRef; -use datatypes::schema::{ColumnSchema, FulltextOptions, Schema, SchemaRef}; +use datatypes::schema::{ColumnSchema, FulltextOptions, Schema, SchemaRef, SkippingIndexOptions}; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize}; use snafu::{ensure, Location, OptionExt, ResultExt, Snafu}; @@ -586,6 +586,10 @@ impl RegionMetadataBuilder { ApiSetIndexOptions::Inverted { column_name } => { self.change_column_inverted_index_options(column_name, true)? } + ApiSetIndexOptions::Skipping { + column_name, + options, + } => self.change_column_skipping_index_options(column_name, Some(options))?, }, AlterKind::UnsetIndex { options } => match options { ApiUnsetIndexOptions::Fulltext { column_name } => { @@ -594,6 +598,9 @@ impl RegionMetadataBuilder { ApiUnsetIndexOptions::Inverted { column_name } => { self.change_column_inverted_index_options(column_name, false)? } + ApiUnsetIndexOptions::Skipping { column_name } => { + self.change_column_skipping_index_options(column_name, None)? + } }, AlterKind::SetRegionOptions { options: _ } => { // nothing to be done with RegionMetadata @@ -764,6 +771,32 @@ impl RegionMetadataBuilder { } Ok(()) } + + fn change_column_skipping_index_options( + &mut self, + column_name: String, + options: Option, + ) -> Result<()> { + for column_meta in self.column_metadatas.iter_mut() { + if column_meta.column_schema.name == column_name { + if let Some(options) = &options { + column_meta + .column_schema + .set_skipping_options(options) + .context(UnsetSkippingIndexOptionsSnafu { + column_name: column_name.clone(), + })?; + } else { + column_meta.column_schema.unset_skipping_options().context( + UnsetSkippingIndexOptionsSnafu { + column_name: column_name.clone(), + }, + )?; + } + } + } + Ok(()) + } } /// Fields skipped in serialization. @@ -919,6 +952,22 @@ pub enum MetadataError { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to set skipping index options for column {}", column_name))] + SetSkippingIndexOptions { + column_name: String, + source: datatypes::Error, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to unset skipping index options for column {}", column_name))] + UnsetSkippingIndexOptions { + column_name: String, + source: datatypes::Error, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for MetadataError { diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs index bdc78f6f7012..b12159975c81 100644 --- a/src/store-api/src/region_request.rs +++ b/src/store-api/src/region_request.rs @@ -17,17 +17,20 @@ use std::fmt::{self, Display}; use api::helper::ColumnDataTypeWrapper; use api::v1::add_column_location::LocationType; -use api::v1::column_def::as_fulltext_option; +use api::v1::column_def::{as_fulltext_option, as_skipping_index_type}; use api::v1::region::{ alter_request, compact_request, region_request, AlterRequest, AlterRequests, CloseRequest, CompactRequest, CreateRequest, CreateRequests, DeleteRequests, DropRequest, DropRequests, FlushRequest, InsertRequests, OpenRequest, TruncateRequest, }; -use api::v1::{self, set_index, Analyzer, Option as PbOption, Rows, SemanticType, WriteHint}; +use api::v1::{ + self, set_index, Analyzer, Option as PbOption, Rows, SemanticType, + SkippingIndexType as PbSkippingIndexType, WriteHint, +}; pub use common_base::AffectedRows; use common_time::TimeToLive; -use datatypes::data_type::ConcreteDataType; -use datatypes::schema::FulltextOptions; +use datatypes::prelude::ConcreteDataType; +use datatypes::schema::{FulltextOptions, SkippingIndexOptions}; use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt, ResultExt}; use strum::{AsRefStr, IntoStaticStr}; @@ -511,6 +514,10 @@ pub enum ApiSetIndexOptions { Inverted { column_name: String, }, + Skipping { + column_name: String, + options: SkippingIndexOptions, + }, } impl ApiSetIndexOptions { @@ -518,6 +525,7 @@ impl ApiSetIndexOptions { match self { ApiSetIndexOptions::Fulltext { column_name, .. } => column_name, ApiSetIndexOptions::Inverted { column_name } => column_name, + ApiSetIndexOptions::Skipping { column_name, .. } => column_name, } } @@ -525,6 +533,7 @@ impl ApiSetIndexOptions { match self { ApiSetIndexOptions::Fulltext { .. } => true, ApiSetIndexOptions::Inverted { .. } => false, + ApiSetIndexOptions::Skipping { .. } => false, } } } @@ -533,6 +542,7 @@ impl ApiSetIndexOptions { pub enum ApiUnsetIndexOptions { Fulltext { column_name: String }, Inverted { column_name: String }, + Skipping { column_name: String }, } impl ApiUnsetIndexOptions { @@ -540,6 +550,7 @@ impl ApiUnsetIndexOptions { match self { ApiUnsetIndexOptions::Fulltext { column_name } => column_name, ApiUnsetIndexOptions::Inverted { column_name } => column_name, + ApiUnsetIndexOptions::Skipping { column_name } => column_name, } } @@ -547,6 +558,7 @@ impl ApiUnsetIndexOptions { match self { ApiUnsetIndexOptions::Fulltext { .. } => true, ApiUnsetIndexOptions::Inverted { .. } => false, + ApiUnsetIndexOptions::Skipping { .. } => false, } } } @@ -722,6 +734,18 @@ impl TryFrom for AlterKind { column_name: i.column_name, }, }, + set_index::Options::Skipping(s) => AlterKind::SetIndex { + options: ApiSetIndexOptions::Skipping { + column_name: s.column_name, + options: SkippingIndexOptions { + index_type: as_skipping_index_type( + PbSkippingIndexType::try_from(s.skipping_index_type) + .context(DecodeProtoSnafu)?, + ), + granularity: s.granularity as u32, + }, + }, + }, }, alter_request::Kind::UnsetIndex(o) => match o.options.unwrap() { v1::unset_index::Options::Fulltext(f) => AlterKind::UnsetIndex { @@ -734,6 +758,11 @@ impl TryFrom for AlterKind { column_name: i.column_name, }, }, + v1::unset_index::Options::Skipping(s) => AlterKind::UnsetIndex { + options: ApiUnsetIndexOptions::Skipping { + column_name: s.column_name, + }, + }, }, }; diff --git a/src/table/src/error.rs b/src/table/src/error.rs index 344140680f35..ef08ebc4a115 100644 --- a/src/table/src/error.rs +++ b/src/table/src/error.rs @@ -156,6 +156,22 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to set skipping index options for column {}", column_name))] + SetSkippingOptions { + column_name: String, + source: datatypes::Error, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to unset skipping index options for column {}", column_name))] + UnsetSkippingOptions { + column_name: String, + source: datatypes::Error, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for Error { @@ -179,7 +195,9 @@ impl ErrorExt for Error { Error::Unsupported { .. } => StatusCode::Unsupported, Error::ParseTableOption { .. } => StatusCode::InvalidArguments, Error::MissingTimeIndexColumn { .. } => StatusCode::IllegalState, - Error::InvalidTableOptionValue { .. } => StatusCode::InvalidArguments, + Error::InvalidTableOptionValue { .. } + | Error::SetSkippingOptions { .. } + | Error::UnsetSkippingOptions { .. } => StatusCode::InvalidArguments, } } diff --git a/src/table/src/metadata.rs b/src/table/src/metadata.rs index c8a65a8407b9..0054afed7d6e 100644 --- a/src/table/src/metadata.rs +++ b/src/table/src/metadata.rs @@ -22,6 +22,7 @@ use datafusion_expr::TableProviderFilterPushDown; pub use datatypes::error::{Error as ConvertError, Result as ConvertResult}; use datatypes::schema::{ ColumnSchema, FulltextOptions, RawSchema, Schema, SchemaBuilder, SchemaRef, + SkippingIndexOptions, }; use derive_builder::Builder; use serde::{Deserialize, Serialize}; @@ -221,6 +222,14 @@ impl TableMeta { SetIndexOptions::Inverted { column_name } => { self.change_column_modify_inverted_index(table_name, column_name, true) } + SetIndexOptions::Skipping { + column_name, + options, + } => self.change_column_skipping_index_options( + table_name, + column_name, + Some(options), + ), }, AlterKind::UnsetIndex { options } => match options { UnsetIndexOptions::Fulltext { column_name } => { @@ -229,6 +238,9 @@ impl TableMeta { UnsetIndexOptions::Inverted { column_name } => { self.change_column_modify_inverted_index(table_name, column_name, false) } + UnsetIndexOptions::Skipping { column_name } => { + self.change_column_skipping_index_options(table_name, column_name, None) + } }, } } @@ -402,6 +414,56 @@ impl TableMeta { Ok(meta_builder) } + /// Creates a [TableMetaBuilder] with modified column skipping index options. + fn change_column_skipping_index_options( + &self, + table_name: &str, + column_name: &str, + options: Option<&SkippingIndexOptions>, + ) -> Result { + let table_schema = &self.schema; + let mut meta_builder = self.new_meta_builder(); + + let mut columns = Vec::with_capacity(table_schema.column_schemas().len()); + for column_schema in table_schema.column_schemas() { + if column_schema.name == column_name { + let mut new_column_schema = column_schema.clone(); + if let Some(options) = options { + set_column_skipping_index_options( + &mut new_column_schema, + column_name, + options, + )?; + } else { + unset_column_skipping_index_options(&mut new_column_schema, column_name)?; + } + columns.push(new_column_schema); + } else { + columns.push(column_schema.clone()); + } + } + + let mut builder = SchemaBuilder::try_from_columns(columns) + .with_context(|_| error::SchemaBuildSnafu { + msg: format!("Failed to convert column schemas into schema for table {table_name}"), + })? + .version(table_schema.version() + 1); + + for (k, v) in table_schema.metadata().iter() { + builder = builder.add_metadata(k, v); + } + + let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu { + msg: format!("Failed to convert column schemas into schema for table {table_name}"), + })?; + + let _ = meta_builder + .schema(Arc::new(new_schema)) + .primary_key_indices(self.primary_key_indices.clone()); + + Ok(meta_builder) + } + // TODO(yingwen): Remove this. /// Allocate a new column for the table. /// @@ -1131,6 +1193,28 @@ fn unset_column_fulltext_options( Ok(()) } +fn set_column_skipping_index_options( + column_schema: &mut ColumnSchema, + column_name: &str, + options: &SkippingIndexOptions, +) -> Result<()> { + column_schema + .set_skipping_options(options) + .context(error::SetSkippingOptionsSnafu { column_name })?; + + Ok(()) +} + +fn unset_column_skipping_index_options( + column_schema: &mut ColumnSchema, + column_name: &str, +) -> Result<()> { + column_schema + .unset_skipping_options() + .context(error::UnsetSkippingOptionsSnafu { column_name })?; + Ok(()) +} + #[cfg(test)] mod tests { use common_error::ext::ErrorExt; diff --git a/src/table/src/requests.rs b/src/table/src/requests.rs index fb435e0e672f..4a409d1293a7 100644 --- a/src/table/src/requests.rs +++ b/src/table/src/requests.rs @@ -25,7 +25,7 @@ use common_time::range::TimestampRange; use common_time::TimeToLive; use datatypes::data_type::ConcreteDataType; use datatypes::prelude::VectorRef; -use datatypes::schema::{ColumnSchema, FulltextOptions}; +use datatypes::schema::{ColumnSchema, FulltextOptions, SkippingIndexOptions}; use greptime_proto::v1::region::compact_request; use serde::{Deserialize, Serialize}; use store_api::metric_engine_consts::{ @@ -239,12 +239,17 @@ pub enum SetIndexOptions { Inverted { column_name: String, }, + Skipping { + column_name: String, + options: SkippingIndexOptions, + }, } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UnsetIndexOptions { Fulltext { column_name: String }, Inverted { column_name: String }, + Skipping { column_name: String }, } #[derive(Debug)] diff --git a/tests/cases/standalone/common/alter/change_col_fulltext_options.result b/tests/cases/standalone/common/alter/change_col_fulltext_options.result index 154e1f8914dd..fc0020afa43c 100644 --- a/tests/cases/standalone/common/alter/change_col_fulltext_options.result +++ b/tests/cases/standalone/common/alter/change_col_fulltext_options.result @@ -42,7 +42,7 @@ SELECT * FROM test WHERE MATCHES(message, 'hello') ORDER BY message; | world hello | 2020-01-02T00:00:01 | +-------------+---------------------+ -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'true'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true'); Affected Rows: 0 @@ -103,7 +103,7 @@ SHOW INDEX FROM test; | test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | +-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ -ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT; +ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT INDEX; Affected Rows: 0 @@ -132,7 +132,7 @@ SHOW INDEX FROM test; | test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | +-------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'true'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true'); Affected Rows: 0 @@ -162,11 +162,11 @@ SHOW INDEX FROM test; | test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | +-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'false'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'false'); Error: 1004(InvalidArguments), Invalid column option, column name: message, error: FULLTEXT index already enabled -ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT; +ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT INDEX; Affected Rows: 0 @@ -195,19 +195,19 @@ SHOW INDEX FROM test; | test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | +-------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinglish', case_sensitive = 'false'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinglish', case_sensitive = 'false'); Error: 1002(Unexpected), Invalid fulltext option: Chinglish, expected: 'English' | 'Chinese' -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'no'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'no'); Error: 1002(Unexpected), Invalid fulltext option: no, expected: 'true' | 'false' -ALTER TABLE test MODIFY COLUMN time SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'false'); +ALTER TABLE test MODIFY COLUMN time SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'false'); Error: 1004(InvalidArguments), Invalid column option, column name: time, error: FULLTEXT index only supports string type -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'English', case_sensitive = 'true'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'true'); Error: 1004(InvalidArguments), Invalid column option, column name: message, error: Cannot change analyzer or case_sensitive if FULLTEXT index is set before. Previous analyzer: Chinese, previous case_sensitive: true diff --git a/tests/cases/standalone/common/alter/change_col_fulltext_options.sql b/tests/cases/standalone/common/alter/change_col_fulltext_options.sql index 06bd601b4f81..b5ead6e6104a 100644 --- a/tests/cases/standalone/common/alter/change_col_fulltext_options.sql +++ b/tests/cases/standalone/common/alter/change_col_fulltext_options.sql @@ -15,7 +15,7 @@ INSERT INTO test VALUES ('hello', '2020-01-01 00:00:00'), SELECT * FROM test WHERE MATCHES(message, 'hello') ORDER BY message; -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'true'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true'); SELECT * FROM test WHERE MATCHES(message, 'hello') ORDER BY message; @@ -31,32 +31,32 @@ SHOW CREATE TABLE test; SHOW INDEX FROM test; -ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT; +ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT INDEX; SHOW CREATE TABLE test; SHOW INDEX FROM test; -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'true'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true'); SHOW CREATE TABLE test; SHOW INDEX FROM test; -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'false'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'false'); -ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT; +ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT INDEX; SHOW CREATE TABLE test; SHOW INDEX FROM test; -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinglish', case_sensitive = 'false'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinglish', case_sensitive = 'false'); -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'no'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'no'); -ALTER TABLE test MODIFY COLUMN time SET FULLTEXT WITH(analyzer = 'Chinese', case_sensitive = 'false'); +ALTER TABLE test MODIFY COLUMN time SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'false'); -ALTER TABLE test MODIFY COLUMN message SET FULLTEXT WITH(analyzer = 'English', case_sensitive = 'true'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'true'); DROP TABLE test; diff --git a/tests/cases/standalone/common/alter/change_col_skipping_options.result b/tests/cases/standalone/common/alter/change_col_skipping_options.result new file mode 100644 index 000000000000..4b6b7705ee3a --- /dev/null +++ b/tests/cases/standalone/common/alter/change_col_skipping_options.result @@ -0,0 +1,263 @@ + -- Test basic skipping index operations on a single column +CREATE TABLE `test` ( + `value` DOUBLE, + `category` STRING, + `metric` INT64, + `time` TIMESTAMP TIME INDEX, +); + +Affected Rows: 0 + +SHOW CREATE TABLE test; + ++-------+-------------------------------------+ +| Table | Create Table | ++-------+-------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "value" DOUBLE NULL, | +| | "category" STRING NULL, | +| | "metric" BIGINT NULL, | +| | "time" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("time") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-------+-------------------------------------+ + +-- Write initial data +INSERT INTO test VALUES +(1.0, 'A', 100, '2020-01-01 00:00:00'), +(2.0, 'B', 200, '2020-01-01 00:00:01'), +(3.0, 'A', 300, '2020-01-02 00:00:00'), +(4.0, 'B', 400, '2020-01-02 00:00:01'); + +Affected Rows: 4 + +-- Test queries before adding skipping index +SELECT * FROM test WHERE value > 2.0 ORDER BY time; + ++-------+----------+--------+---------------------+ +| value | category | metric | time | ++-------+----------+--------+---------------------+ +| 3.0 | A | 300 | 2020-01-02T00:00:00 | +| 4.0 | B | 400 | 2020-01-02T00:00:01 | ++-------+----------+--------+---------------------+ + +SELECT * FROM test WHERE metric > 200 ORDER BY time; + ++-------+----------+--------+---------------------+ +| value | category | metric | time | ++-------+----------+--------+---------------------+ +| 3.0 | A | 300 | 2020-01-02T00:00:00 | +| 4.0 | B | 400 | 2020-01-02T00:00:01 | ++-------+----------+--------+---------------------+ + +-- Add skipping index +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(granularity = 1024, type = 'BLOOM'); + +Affected Rows: 0 + +-- Test queries after adding skipping index +SELECT * FROM test WHERE value > 2.0 ORDER BY time; + ++-------+----------+--------+---------------------+ +| value | category | metric | time | ++-------+----------+--------+---------------------+ +| 3.0 | A | 300 | 2020-01-02T00:00:00 | +| 4.0 | B | 400 | 2020-01-02T00:00:01 | ++-------+----------+--------+---------------------+ + +SELECT * FROM test WHERE value BETWEEN 2.0 AND 4.0 ORDER BY time; + ++-------+----------+--------+---------------------+ +| value | category | metric | time | ++-------+----------+--------+---------------------+ +| 2.0 | B | 200 | 2020-01-01T00:00:01 | +| 3.0 | A | 300 | 2020-01-02T00:00:00 | +| 4.0 | B | 400 | 2020-01-02T00:00:01 | ++-------+----------+--------+---------------------+ + +-- Add more data to test dynamic updates +INSERT INTO test VALUES +(5.0, 'C', 500, '2020-01-03 00:00:00'), +(6.0, 'A', 600, '2020-01-03 00:00:01'), +(7.0, 'B', 700, '2020-01-04 00:00:00'), +(8.0, 'C', 800, '2020-01-04 00:00:01'); + +Affected Rows: 4 + +-- Test queries with new data +SELECT * FROM test WHERE value > 6.0 ORDER BY time; + ++-------+----------+--------+---------------------+ +| value | category | metric | time | ++-------+----------+--------+---------------------+ +| 7.0 | B | 700 | 2020-01-04T00:00:00 | +| 8.0 | C | 800 | 2020-01-04T00:00:01 | ++-------+----------+--------+---------------------+ + +SELECT * FROM test WHERE value < 3.0 ORDER BY time; + ++-------+----------+--------+---------------------+ +| value | category | metric | time | ++-------+----------+--------+---------------------+ +| 1.0 | A | 100 | 2020-01-01T00:00:00 | +| 2.0 | B | 200 | 2020-01-01T00:00:01 | ++-------+----------+--------+---------------------+ + +-- Test multiple columns with skipping indexes +ALTER TABLE test MODIFY COLUMN metric SET SKIPPING INDEX WITH(granularity = 1024, type = 'BLOOM'); + +Affected Rows: 0 + +-- Test queries with multiple skipping indexes +SELECT * FROM test WHERE value > 5.0 AND metric < 700 ORDER BY time; + ++-------+----------+--------+---------------------+ +| value | category | metric | time | ++-------+----------+--------+---------------------+ +| 6.0 | A | 600 | 2020-01-03T00:00:01 | ++-------+----------+--------+---------------------+ + +-- SQLNESS ARG restart=true +-- Verify persistence after restart +SHOW CREATE TABLE test; + ++-------+-----------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+-----------------------------------------------------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "value" DOUBLE NULL SKIPPING INDEX WITH(granularity = '1024', type = 'BLOOM'), | +| | "category" STRING NULL, | +| | "metric" BIGINT NULL SKIPPING INDEX WITH(granularity = '1024', type = 'BLOOM'), | +| | "time" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("time") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-------+-----------------------------------------------------------------------------------+ + +SHOW INDEX FROM test; + ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ +| test | 1 | SKIPPING INDEX | 3 | metric | A | | | | YES | greptime-bloom-filter-v1 | | | YES | | +| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | +| test | 1 | SKIPPING INDEX | 1 | value | A | | | | YES | greptime-bloom-filter-v1 | | | YES | | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ + +-- Test modifying existing skipping index options +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(granularity = 8192, type = 'BLOOM'); + +Affected Rows: 0 + +SHOW CREATE TABLE test; + ++-------+-----------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+-----------------------------------------------------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "value" DOUBLE NULL SKIPPING INDEX WITH(granularity = '8192', type = 'BLOOM'), | +| | "category" STRING NULL, | +| | "metric" BIGINT NULL SKIPPING INDEX WITH(granularity = '1024', type = 'BLOOM'), | +| | "time" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("time") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-------+-----------------------------------------------------------------------------------+ + +-- Test removing skipping index +ALTER TABLE test MODIFY COLUMN value UNSET SKIPPING INDEX; + +Affected Rows: 0 + +SHOW INDEX FROM test; + ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ +| test | 1 | SKIPPING INDEX | 3 | metric | A | | | | YES | greptime-bloom-filter-v1 | | | YES | | +| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ + +-- Test adding back with different options +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(granularity = 2048, type = 'BLOOM'); + +Affected Rows: 0 + +SHOW CREATE TABLE test; + ++-------+-----------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+-----------------------------------------------------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "value" DOUBLE NULL SKIPPING INDEX WITH(granularity = '2048', type = 'BLOOM'), | +| | "category" STRING NULL, | +| | "metric" BIGINT NULL SKIPPING INDEX WITH(granularity = '1024', type = 'BLOOM'), | +| | "time" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("time") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-------+-----------------------------------------------------------------------------------+ + +-- Test removing all skipping indexes +ALTER TABLE test MODIFY COLUMN value UNSET SKIPPING INDEX; + +Affected Rows: 0 + +ALTER TABLE test MODIFY COLUMN metric UNSET SKIPPING INDEX; + +Affected Rows: 0 + +SHOW INDEX FROM test; + ++-------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ +| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | ++-------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ + +-- Test invalid operations and error cases +-- Try to set skipping index on string column (should fail) +ALTER TABLE test MODIFY COLUMN category SET SKIPPING INDEX WITH(granularity = 1024, type = 'BLOOM'); + +Affected Rows: 0 + +-- Try to set skipping index on timestamp column (should fail) +ALTER TABLE test MODIFY COLUMN time SET SKIPPING INDEX WITH(granularity = 1024, type = 'BLOOM'); + +Affected Rows: 0 + +-- Test invalid option values +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(blabla = 1024, type = 'BLOOM'); + +Error: 1004(InvalidArguments), Invalid column option, column name: value, error: invalid SKIPPING INDEX option: blabla + +-- Test partial options +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(granularity = 4096); + +Affected Rows: 0 + +SHOW INDEX FROM test; + ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ +| test | 1 | SKIPPING INDEX | 2 | category | A | | | | YES | greptime-bloom-filter-v1 | | | YES | | +| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | +| test | 1 | SKIPPING INDEX | 4 | time | A | | | | YES | greptime-bloom-filter-v1 | | | YES | | +| test | 1 | SKIPPING INDEX | 1 | value | A | | | | YES | greptime-bloom-filter-v1 | | | YES | | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+--------------------------+---------+---------------+---------+------------+ + +-- Clean up +DROP TABLE test; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/alter/change_col_skipping_options.sql b/tests/cases/standalone/common/alter/change_col_skipping_options.sql new file mode 100644 index 000000000000..905a61598381 --- /dev/null +++ b/tests/cases/standalone/common/alter/change_col_skipping_options.sql @@ -0,0 +1,83 @@ + -- Test basic skipping index operations on a single column +CREATE TABLE `test` ( + `value` DOUBLE, + `category` STRING, + `metric` INT64, + `time` TIMESTAMP TIME INDEX, +); + +SHOW CREATE TABLE test; + +-- Write initial data +INSERT INTO test VALUES +(1.0, 'A', 100, '2020-01-01 00:00:00'), +(2.0, 'B', 200, '2020-01-01 00:00:01'), +(3.0, 'A', 300, '2020-01-02 00:00:00'), +(4.0, 'B', 400, '2020-01-02 00:00:01'); + +-- Test queries before adding skipping index +SELECT * FROM test WHERE value > 2.0 ORDER BY time; +SELECT * FROM test WHERE metric > 200 ORDER BY time; + +-- Add skipping index +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(granularity = 1024, type = 'BLOOM'); + +-- Test queries after adding skipping index +SELECT * FROM test WHERE value > 2.0 ORDER BY time; +SELECT * FROM test WHERE value BETWEEN 2.0 AND 4.0 ORDER BY time; + +-- Add more data to test dynamic updates +INSERT INTO test VALUES +(5.0, 'C', 500, '2020-01-03 00:00:00'), +(6.0, 'A', 600, '2020-01-03 00:00:01'), +(7.0, 'B', 700, '2020-01-04 00:00:00'), +(8.0, 'C', 800, '2020-01-04 00:00:01'); + +-- Test queries with new data +SELECT * FROM test WHERE value > 6.0 ORDER BY time; +SELECT * FROM test WHERE value < 3.0 ORDER BY time; + +-- Test multiple columns with skipping indexes +ALTER TABLE test MODIFY COLUMN metric SET SKIPPING INDEX WITH(granularity = 1024, type = 'BLOOM'); + +-- Test queries with multiple skipping indexes +SELECT * FROM test WHERE value > 5.0 AND metric < 700 ORDER BY time; + +-- SQLNESS ARG restart=true +-- Verify persistence after restart +SHOW CREATE TABLE test; +SHOW INDEX FROM test; + +-- Test modifying existing skipping index options +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(granularity = 8192, type = 'BLOOM'); +SHOW CREATE TABLE test; + +-- Test removing skipping index +ALTER TABLE test MODIFY COLUMN value UNSET SKIPPING INDEX; +SHOW INDEX FROM test; + +-- Test adding back with different options +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(granularity = 2048, type = 'BLOOM'); +SHOW CREATE TABLE test; + +-- Test removing all skipping indexes +ALTER TABLE test MODIFY COLUMN value UNSET SKIPPING INDEX; +ALTER TABLE test MODIFY COLUMN metric UNSET SKIPPING INDEX; +SHOW INDEX FROM test; + +-- Test invalid operations and error cases +-- Try to set skipping index on string column (should fail) +ALTER TABLE test MODIFY COLUMN category SET SKIPPING INDEX WITH(granularity = 1024, type = 'BLOOM'); + +-- Try to set skipping index on timestamp column (should fail) +ALTER TABLE test MODIFY COLUMN time SET SKIPPING INDEX WITH(granularity = 1024, type = 'BLOOM'); + +-- Test invalid option values +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(blabla = 1024, type = 'BLOOM'); + +-- Test partial options +ALTER TABLE test MODIFY COLUMN value SET SKIPPING INDEX WITH(granularity = 4096); +SHOW INDEX FROM test; + +-- Clean up +DROP TABLE test;