Skip to content

Commit

Permalink
Time ranges
Browse files Browse the repository at this point in the history
  • Loading branch information
@umeshma
umeshma committed Jan 31, 2025
1 parent 30029d7 commit 22e875b
Show file tree
Hide file tree
Showing 5 changed files with 96 additions and 75 deletions.
25 changes: 8 additions & 17 deletions ts/packages/knowPro/src/accumulators.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,19 +144,6 @@ export class MatchAccumulator<T = any> {
return topN.length;
}

public union(other: MatchAccumulator<T>): void {
for (const matchFrom of other.matches.values()) {
const matchTo = this.matches.get(matchFrom.value);
if (matchTo !== undefined) {
// Existing
matchTo.hitCount += matchFrom.hitCount;
matchTo.score += matchFrom.score;
} else {
this.matches.set(matchFrom.value, matchFrom);
}
}
}

private matchesWithMinHitCount(
minHitCount: number | undefined,
): IterableIterator<Match<T>> {
Expand Down Expand Up @@ -282,10 +269,8 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
}

private getMinHitCount(minHitCount?: number): number {
return minHitCount !== undefined
? minHitCount
: //: this.queryTermMatches.termMatches.size;
this.maxHits;
return minHitCount !== undefined ? minHitCount : this.maxHits;
//: this.queryTermMatches.termMatches.size;
}
}

Expand Down Expand Up @@ -346,6 +331,12 @@ export class TextRangeAccumulator {
textRanges.push(textRange);
}

public addRanges(textRanges: TextRange[]) {
for (const range of textRanges) {
this.addRange(range);
}
}

public isInRange(textRange: TextRange): boolean {
const textRanges = this.rangesForMessage.get(
textRange.start.messageIndex,
Expand Down
2 changes: 1 addition & 1 deletion ts/packages/knowPro/src/dataFormat.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,5 +146,5 @@ export type DateRange = {
};

export interface ITimestampToMessageIndex {
getMessagesInDateRange(dateRange: DateRange): MessageIndex[];
getTextRange(dateRange: DateRange): TextRange[];
}
72 changes: 40 additions & 32 deletions ts/packages/knowPro/src/query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import {
IConversation,
IMessage,
ITag,
ITermToRelatedTermsIndex,
ITermToSemanticRefIndex,
ITopic,
KnowledgeType,
Expand Down Expand Up @@ -95,13 +94,13 @@ export function isInTextRange(
innerRange: TextRange,
): boolean {
// outer start must be <= inner start
// inner end must be <= outerEnd
// inner end must be < outerEnd (which is exclusive)
let cmpStart = compareTextLocation(outerRange.start, innerRange.start);
let cmpEnd = compareTextLocation(
innerRange.end ?? MaxTextLocation,
outerRange.end ?? MaxTextLocation,
);
return cmpStart <= 0 && cmpEnd <= 0;
return cmpStart <= 0 && cmpEnd < 0;
}

export function compareDates(x: Date, y: Date): number {
Expand All @@ -124,27 +123,14 @@ export interface IQueryOpExpr<T> {
}

export class QueryEvalContext {
constructor(private conversation: IConversation) {
constructor(public conversation: IConversation) {
if (!isConversationSearchable(conversation)) {
throw new Error(`${conversation.nameTag} is not initialized`);
}
}

public get semanticRefIndex(): ITermToSemanticRefIndex {
this.conversation.messages;
return this.conversation.semanticRefIndex!;
}

public get semanticRefs(): SemanticRef[] {
return this.conversation.semanticRefs!;
}

public get relatedTermIndex(): ITermToRelatedTermsIndex | undefined {
return this.conversation.relatedTermsIndex;
}

public getSemanticRef(semanticRefIndex: SemanticRefIndex): SemanticRef {
return this.semanticRefs[semanticRefIndex];
return this.conversation.semanticRefs![semanticRefIndex];
}

public getMessageForRef(semanticRef: SemanticRef): IMessage {
Expand Down Expand Up @@ -177,10 +163,12 @@ export class TermsMatchExpr implements IQueryOpExpr<SemanticRefAccumulator> {
): Promise<SemanticRefAccumulator> {
const matchAccumulator: SemanticRefAccumulator =
new SemanticRefAccumulator();
const index = context.semanticRefIndex;
const terms = await this.terms.eval(context);
for (const queryTerm of terms) {
this.accumulateMatches(index, matchAccumulator, queryTerm);
const index = context.conversation.semanticRefIndex;
if (index !== undefined) {
const terms = await this.terms.eval(context);
for (const queryTerm of terms) {
this.accumulateMatches(index, matchAccumulator, queryTerm);
}
}
return Promise.resolve(matchAccumulator);
}
Expand Down Expand Up @@ -214,7 +202,7 @@ export class ResolveRelatedTermsExpr implements IQueryOpExpr<QueryTerm[]> {

public async eval(context: QueryEvalContext): Promise<QueryTerm[]> {
const terms = await this.terms.eval(context);
const index = context.relatedTermIndex;
const index = context.conversation.relatedTermsIndex;
if (index !== undefined) {
for (const queryTerm of terms) {
if (
Expand Down Expand Up @@ -253,7 +241,7 @@ export class GroupByKnowledgeTypeExpr
): Promise<Map<KnowledgeType, SemanticRefAccumulator>> {
const semanticRefMatches = await this.matches.eval(context);
return semanticRefMatches.groupMatchesByKnowledgeType(
context.semanticRefs,
context.conversation.semanticRefs!,
);
}
}
Expand Down Expand Up @@ -531,31 +519,38 @@ function matchTextOneOf(
export class ScopeExpr implements IQueryOpExpr<SemanticRefAccumulator> {
constructor(
public sourceExpr: IQueryOpExpr<SemanticRefAccumulator>,
// Predicates that identity what is in scope
// Predicates that look at matched semantic refs to determine what is in scope
public predicates: IQuerySemanticRefPredicate[],
public scopeExpr: IQueryOpExpr<TextRange[]> | undefined = undefined,
) {}

public async eval(
context: QueryEvalContext,
): Promise<SemanticRefAccumulator> {
let accumulator = await this.sourceExpr.eval(context);
const tagScope = new TextRangeAccumulator();
const scope = new TextRangeAccumulator();
if (this.scopeExpr !== undefined) {
const timeRanges = await this.scopeExpr.eval(context);
if (timeRanges !== undefined) {
scope.addRanges(timeRanges);
}
}
for (const inScopeRef of accumulator.getSemanticRefs(
context.semanticRefs,
context.conversation.semanticRefs!,
(sr) =>
this.evalPredicates(
context,
accumulator.queryTermMatches,
this.predicates,
this.predicates!,
sr,
),
)) {
tagScope.addRange(inScopeRef.range);
scope.addRange(inScopeRef.range);
}
if (tagScope.size > 0) {
if (scope.size > 0) {
accumulator = accumulator.selectInScope(
context.semanticRefs,
tagScope,
context.conversation.semanticRefs!,
scope,
);
}
return Promise.resolve(accumulator);
Expand All @@ -575,3 +570,16 @@ export class ScopeExpr implements IQueryOpExpr<SemanticRefAccumulator> {
return false;
}
}

export class TimestampScopeExpr implements IQueryOpExpr<TextRange[]> {
constructor(public dateRange: DateRange) {}

public eval(context: QueryEvalContext): Promise<TextRange[]> {
const index = context.conversation.timestampIndex;
let textRanges: TextRange[] | undefined;
if (index !== undefined) {
textRanges = index.getTextRange(this.dateRange);
}
return Promise.resolve(textRanges ?? []);
}
}
19 changes: 16 additions & 3 deletions ts/packages/knowPro/src/search.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import { SemanticRefAccumulator } from "./accumulators.js";
import {
DateRange,
IConversation,
KnowledgeType,
QueryTerm,
Expand All @@ -19,6 +20,7 @@ export type SearchResult = {
export type SearchFilter = {
type?: KnowledgeType | undefined;
propertiesToMatch?: Record<string, string>;
dateRange?: DateRange;
};
/**
* Searches conversation for terms
Expand Down Expand Up @@ -93,9 +95,7 @@ class SearchQueryBuilder {
: queryTerms,
);
// Always apply "tag match" scope... all text ranges that matched tags.. are in scope
termsMatchExpr = new q.ScopeExpr(termsMatchExpr, [
new q.KnowledgeTypePredicate("tag"),
]);
termsMatchExpr = this.compileScope(termsMatchExpr, filter?.dateRange);
if (filter !== undefined) {
// Where clause
termsMatchExpr = new q.WhereSemanticRefExpr(
Expand All @@ -106,6 +106,19 @@ class SearchQueryBuilder {
return termsMatchExpr;
}

private compileScope(
termsMatchExpr: q.IQueryOpExpr<SemanticRefAccumulator>,
dateRange?: DateRange,
): q.IQueryOpExpr<SemanticRefAccumulator> {
// Always apply "tag match" scope... all text ranges that matched tags.. are in scope
termsMatchExpr = new q.ScopeExpr(
termsMatchExpr,
[new q.KnowledgeTypePredicate("tag")],
dateRange ? new q.TimestampScopeExpr(dateRange) : undefined,
);
return termsMatchExpr;
}

private compileFilter(
filter: SearchFilter,
): q.IQuerySemanticRefPredicate[] {
Expand Down
53 changes: 31 additions & 22 deletions ts/packages/knowPro/src/timestampIndex.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,31 @@ import {
IMessage,
ITimestampToMessageIndex,
MessageIndex,
TextRange,
} from "./dataFormat.js";

export class TimestampToMessageIndex implements ITimestampToMessageIndex {
private messageIndex: Timestamped<MessageIndex>[];
private messageIndex: TimestampedTextRange[];
constructor(messages: IMessage[]) {
this.messageIndex = [];
for (let i = 0; i < messages.length; ++i) {
this.addMessage(messages[i], i);
}
this.messageIndex.sort(compareTimestamped);
this.messageIndex.sort(compareTimestampedRange);
}

public getMessagesInDateRange(dateRange: DateRange): MessageIndex[] {
return collections.getInRange(
public getTextRange(dateRange: DateRange): TextRange[] {
const startAt = dateTime.timestampString(dateRange.start);
const stopAt = dateRange.end
? dateTime.timestampString(dateRange.end)
: undefined;
const ranges: TimestampedTextRange[] = collections.getInRange(
this.messageIndex,
dateTime.timestampString(dateRange.start),
dateRange.end ? dateTime.timestampString(dateRange.end) : undefined,
compareTimestamped,
startAt,
stopAt,
compareTimestampedRange,
);
return ranges.map((r) => r.range);
}

private addMessage(
Expand All @@ -38,35 +44,38 @@ export class TimestampToMessageIndex implements ITimestampToMessageIndex {
}
const date = new Date(message.timestamp);
// This string is formatted to be searchable
const entry: Timestamped<MessageIndex> = makeTimestamped(
date,
messageIndex,
);
const entry = this.makeTimestamped(date, messageIndex);
if (inOrder) {
collections.insertIntoSorted(
this.messageIndex,
entry,
compareTimestamped,
compareTimestampedRange,
);
} else {
this.messageIndex.push(entry);
}
return true;
}

private makeTimestamped(
timestamp: Date,
messageIndex: MessageIndex,
): TimestampedTextRange {
return {
range: { start: { messageIndex } },
timestamp: dateTime.timestampString(timestamp, false),
};
}
}

type Timestamped<T = any> = {
type TimestampedTextRange = {
timestamp: string;
value: T;
range: TextRange;
};

function compareTimestamped(x: Timestamped, y: Timestamped) {
function compareTimestampedRange(
x: TimestampedTextRange,
y: TimestampedTextRange,
) {
return x.timestamp.localeCompare(y.timestamp);
}

function makeTimestamped(timestamp: Date, value: any): Timestamped {
return {
value,
timestamp: dateTime.timestampString(timestamp, false),
};
}

0 comments on commit 22e875b

Please sign in to comment.