From ff43892ef45b03249fa8d22ef993f433f6889633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 17 Apr 2019 11:40:37 -0400 Subject: [PATCH 01/32] - generic configuration for lexer - this is for making lexemes type aware --- src/core/syntax-tree/lexer.ts | 26 ++++++++++++++------- src/core/syntax-tree/lexicon.ts | 16 +++++++++---- src/dash-table/syntax-tree/lexicon/query.ts | 4 +++- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/core/syntax-tree/lexer.ts b/src/core/syntax-tree/lexer.ts index c0fa303f5..95c713a52 100644 --- a/src/core/syntax-tree/lexer.ts +++ b/src/core/syntax-tree/lexer.ts @@ -2,19 +2,27 @@ import * as R from 'ramda'; import { ILexeme, Lexicon } from 'core/syntax-tree/lexicon'; -export interface ILexerResult { - lexemes: ILexemeResult[]; +export interface ILexerResult { + lexemes: ILexemeResult[]; valid: boolean; error?: string; } -export interface ILexemeResult { - lexeme: ILexeme; +export interface ILexemeResult { + lexeme: ILexeme; value?: string; } -export default function lexer(lexicon: Lexicon, query: string): ILexerResult { - let result: ILexemeResult[] = []; +export default function lexer(lexicon: Lexicon, query: string) { + return configurableLexer(lexicon, query, undefined); +} + +export function configurableLexer( + lexicon: Lexicon, + query: string, + config: TConfig +): ILexerResult { + let result: ILexemeResult[] = []; while (query.length) { query = query.replace(/^\s+/, ''); @@ -22,10 +30,10 @@ export default function lexer(lexicon: Lexicon, query: string): ILexerResult { const previous = result.slice(-1)[0]; const previousLexeme = previous ? previous.lexeme : null; - let lexemes: ILexeme[] = lexicon.filter(lexeme => + let lexemes: ILexeme[] = lexicon.filter(lexeme => lexeme.if && (!Array.isArray(lexeme.if) ? - lexeme.if(result, previous) : + lexeme.if(result, previous, config) : (previousLexeme ? lexeme.if && lexeme.if.indexOf(previousLexeme.type) !== -1 : lexeme.if && lexeme.if.indexOf(undefined) !== -1)) @@ -45,7 +53,7 @@ export default function lexer(lexicon: Lexicon, query: string): ILexerResult { const last = result.slice(-1)[0]; const terminal: boolean = last && (typeof last.lexeme.terminal === 'function' ? - last.lexeme.terminal(result, last) : + last.lexeme.terminal(result, last, config) : last.lexeme.terminal); return { diff --git a/src/core/syntax-tree/lexicon.ts b/src/core/syntax-tree/lexicon.ts index 6a26e1b77..96b54138d 100644 --- a/src/core/syntax-tree/lexicon.ts +++ b/src/core/syntax-tree/lexicon.ts @@ -24,13 +24,21 @@ export interface IUnboundedLexeme { syntaxer?: (lexs: any[], pivot: any, pivotIndex: number) => any; } -export interface ILexeme extends IUnboundedLexeme { - terminal: boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); - if: (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); +type Terminal = boolean | + ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean) | + ((lexemes: ILexemeResult[], previous: ILexemeResult, config: TConfig) => boolean); + +type If = (string | undefined)[] | + ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean) | + ((lexemes: ILexemeResult[], previous: ILexemeResult, config: TConfig) => boolean); + +export interface ILexeme extends IUnboundedLexeme { + terminal: Terminal; + if: If; } export function boundLexeme(lexeme: IUnboundedLexeme) { return { ...lexeme, if: () => false, terminal: false }; } -export type Lexicon = ILexeme[]; \ No newline at end of file +export type Lexicon = ILexeme[]; \ No newline at end of file diff --git a/src/dash-table/syntax-tree/lexicon/query.ts b/src/dash-table/syntax-tree/lexicon/query.ts index 8b23c1b63..9a8fca8ea 100644 --- a/src/dash-table/syntax-tree/lexicon/query.ts +++ b/src/dash-table/syntax-tree/lexicon/query.ts @@ -67,7 +67,7 @@ const ifOperator = (_: ILexemeResult[], previous: ILexemeResult) => [LexemeType.Operand] ); -const lexicon: ILexeme[] = [ +const untypedQueryLexicon = [ { ...and, if: ifLogicalOperator, @@ -165,4 +165,6 @@ const lexicon: ILexeme[] = [ })) ]; +const lexicon: ILexeme[] = untypedQueryLexicon; + export default lexicon; \ No newline at end of file From 8d5c5ad6b12b9129e1847f8c7ebb6524da16de01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 17 Apr 2019 12:17:59 -0400 Subject: [PATCH 02/32] - configurable syntax tree - fixing ts typing issues --- package.json | 4 ++-- src/core/syntax-tree/index.ts | 13 +++++++------ src/core/syntax-tree/syntaxer.ts | 6 +++--- src/dash-table/components/FilterFactory.tsx | 3 ++- src/dash-table/derived/cell/dropdowns.ts | 2 +- src/dash-table/derived/cell/wrapperStyles.ts | 2 +- src/dash-table/derived/filter/wrapperStyles.ts | 2 +- src/dash-table/derived/header/wrapperStyles.ts | 2 +- .../syntax-tree/MultiColumnsSyntaxTree.ts | 11 +++++++++-- src/dash-table/syntax-tree/QuerySyntaxTree.ts | 2 +- .../syntax-tree/SingleColumnSyntaxTree.ts | 2 +- src/dash-table/syntax-tree/index.ts | 2 +- 12 files changed, 30 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 93c1f639b..49601623f 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "@storybook/react": "^5.0.5", "@types/d3-format": "^1.3.1", "@types/papaparse": "^4.5.9", - "@types/ramda": "^0.26.5", + "@types/ramda": "^0.26.6", "@types/react": "^16.8.8", "@types/react-dom": "^16.8.3", "@types/react-select": "^1.3.4", @@ -82,7 +82,7 @@ "style-loader": "^0.23.1", "ts-loader": "^5.3.3", "tslint": "^5.14.0", - "typescript": "^3.3.4000", + "typescript": "^3.4.3", "wait-on": "^3.2.0", "webpack": "^4.29.6", "webpack-cli": "^3.3.0", diff --git a/src/core/syntax-tree/index.ts b/src/core/syntax-tree/index.ts index bf3e3568f..4ad325a39 100644 --- a/src/core/syntax-tree/index.ts +++ b/src/core/syntax-tree/index.ts @@ -1,7 +1,7 @@ import * as R from 'ramda'; import Logger from 'core/Logger'; -import lexer, { ILexerResult } from 'core/syntax-tree/lexer'; +import { ILexerResult, configurableLexer } from 'core/syntax-tree/lexer'; import syntaxer, { ISyntaxerResult, ISyntaxTree } from 'core/syntax-tree/syntaxer'; import { Lexicon } from './lexicon'; @@ -39,8 +39,8 @@ function toStructure(tree: ISyntaxTree): IStructure { return res; } -export default class SyntaxTree { - protected lexerResult: ILexerResult; +export default class SyntaxTree { + protected lexerResult: ILexerResult; protected syntaxerResult: ISyntaxerResult; get isValid() { @@ -52,11 +52,12 @@ export default class SyntaxTree { } constructor( - public readonly lexicon: Lexicon, + public readonly lexicon: Lexicon, public readonly query: string, - postProcessor: (res: ILexerResult) => ILexerResult = res => res + public readonly config: TConfig, + postProcessor: (res: ILexerResult) => ILexerResult = res => res ) { - this.lexerResult = postProcessor(lexer(this.lexicon, this.query)); + this.lexerResult = postProcessor(configurableLexer(this.lexicon, this.query, this.config)); this.syntaxerResult = syntaxer(this.lexerResult); } diff --git a/src/core/syntax-tree/syntaxer.ts b/src/core/syntax-tree/syntaxer.ts index d420747e0..b5507808f 100644 --- a/src/core/syntax-tree/syntaxer.ts +++ b/src/core/syntax-tree/syntaxer.ts @@ -9,14 +9,14 @@ export interface ISyntaxerResult { } export interface ISyntaxTree { - lexeme: ILexeme; + lexeme: ILexeme; block?: ISyntaxTree; left?: ISyntaxTree; right: ISyntaxTree; value: string; } -const parser = (lexs: ILexemeResult[]): ISyntaxTree => { +const parser = (lexs: ILexemeResult[]): ISyntaxTree => { let nesting = 0; const nestedLexs = lexs.map(lex => { @@ -57,7 +57,7 @@ const parser = (lexs: ILexemeResult[]): ISyntaxTree => { } }; -export default (lexerResult: ILexerResult): ISyntaxerResult => { +export default (lexerResult: ILexerResult): ISyntaxerResult => { const { lexemes } = lexerResult; if (!lexerResult.valid) { diff --git a/src/dash-table/components/FilterFactory.tsx b/src/dash-table/components/FilterFactory.tsx index 1aa02ecf5..20695e2e6 100644 --- a/src/dash-table/components/FilterFactory.tsx +++ b/src/dash-table/components/FilterFactory.tsx @@ -13,6 +13,7 @@ import derivedHeaderOperations from 'dash-table/derived/header/operations'; import { derivedRelevantFilterStyles } from 'dash-table/derived/style'; import { BasicFilters, Cells, Style } from 'dash-table/derived/style/props'; import { MultiColumnsSyntaxTree, SingleColumnSyntaxTree, getMultiColumnQueryString, getSingleColumnMap } from 'dash-table/syntax-tree'; +import SyntaxTree from 'core/syntax-tree'; type SetFilter = (filter: string, rawFilter: string) => void; @@ -65,7 +66,7 @@ export default class FilterFactory { const rawGlobalFilter = R.map( ast => ast.query || '', - R.filter(ast => Boolean(ast), asts) + R.filter(ast => Boolean(ast), asts) ).join(' && '); setFilter(globalFilter, rawGlobalFilter); diff --git a/src/dash-table/derived/cell/dropdowns.ts b/src/dash-table/derived/cell/dropdowns.ts index 2f8e81ef1..64961c99f 100644 --- a/src/dash-table/derived/cell/dropdowns.ts +++ b/src/dash-table/derived/cell/dropdowns.ts @@ -96,7 +96,7 @@ class Dropdowns { ...(staticDropdown ? [staticDropdown] : []), ...R.map( ([cd]) => cd.dropdown, - R.filter( + R.filter<[IConditionalDropdown, number]>( ([cd, i]) => this.evaluation.get(column.id, i)( this.ast.get(column.id, i)(cd.condition), datum diff --git a/src/dash-table/derived/cell/wrapperStyles.ts b/src/dash-table/derived/cell/wrapperStyles.ts index d7ead3d7d..f53d78d98 100644 --- a/src/dash-table/derived/cell/wrapperStyles.ts +++ b/src/dash-table/derived/cell/wrapperStyles.ts @@ -16,7 +16,7 @@ function getter( return R.addIndex(R.map)((datum, index) => R.map(column => { const relevantStyles = R.map( s => s.style, - R.filter( + R.filter( style => style.matchesColumn(column) && style.matchesRow(index + offset.rows) && diff --git a/src/dash-table/derived/filter/wrapperStyles.ts b/src/dash-table/derived/filter/wrapperStyles.ts index 5cba4a033..81fa97ce2 100644 --- a/src/dash-table/derived/filter/wrapperStyles.ts +++ b/src/dash-table/derived/filter/wrapperStyles.ts @@ -16,7 +16,7 @@ function getter( return R.map(column => { const relevantStyles = R.map( s => s.style, - R.filter( + R.filter( style => style.matchesColumn(column), filterStyles ) diff --git a/src/dash-table/derived/header/wrapperStyles.ts b/src/dash-table/derived/header/wrapperStyles.ts index e90a58f03..d887da1f6 100644 --- a/src/dash-table/derived/header/wrapperStyles.ts +++ b/src/dash-table/derived/header/wrapperStyles.ts @@ -17,7 +17,7 @@ function getter( return R.map(idx => R.map(column => { const relevantStyles = R.map( s => s.style, - R.filter( + R.filter( style => style.matchesColumn(column) && style.matchesRow(idx), diff --git a/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts b/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts index e492e1576..6871867bb 100644 --- a/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts +++ b/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts @@ -5,10 +5,11 @@ import { LexemeType } from 'core/syntax-tree/lexicon'; import { ISyntaxTree } from 'core/syntax-tree/syntaxer'; import columnMultiLexicon from './lexicon/columnMulti'; +import { ILexemeResult } from 'core/syntax-tree/lexer'; export default class MultiColumnsSyntaxTree extends SyntaxTree { constructor(query: string) { - super(columnMultiLexicon, query); + super(columnMultiLexicon, query, undefined); } get isValid() { return super.isValid && @@ -39,7 +40,13 @@ export default class MultiColumnsSyntaxTree extends SyntaxTree { return statements; } private respectsBasicSyntax() { - const fields = R.map(item => item.value, R.filter(i => i.lexeme.type === LexemeType.Operand, this.lexerResult.lexemes)); + const fields = R.map( + (item: ILexemeResult) => item.value, + R.filter( + i => i.lexeme.type === LexemeType.Operand, + this.lexerResult.lexemes + ) + ); const uniqueFields = R.uniq(fields); return fields.length === uniqueFields.length; } diff --git a/src/dash-table/syntax-tree/QuerySyntaxTree.ts b/src/dash-table/syntax-tree/QuerySyntaxTree.ts index a309b9d2d..58286a5a5 100644 --- a/src/dash-table/syntax-tree/QuerySyntaxTree.ts +++ b/src/dash-table/syntax-tree/QuerySyntaxTree.ts @@ -4,6 +4,6 @@ import queryLexicon from './lexicon/query'; export default class QuerySyntaxTree extends SyntaxTree { constructor(query: string) { - super(queryLexicon, query); + super(queryLexicon, query, undefined); } } \ No newline at end of file diff --git a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts index a2d2b7076..6ed70aa9f 100644 --- a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts +++ b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts @@ -46,6 +46,6 @@ export function modifyLex(key: ColumnId, res: ILexerResult) { export default class SingleColumnSyntaxTree extends SyntaxTree { constructor(key: ColumnId, query: string) { - super(columnLexicon, query, modifyLex.bind(undefined, key)); + super(columnLexicon, query, undefined, modifyLex.bind(undefined, key)); } } \ No newline at end of file diff --git a/src/dash-table/syntax-tree/index.ts b/src/dash-table/syntax-tree/index.ts index a651fbef0..ada1e35a3 100644 --- a/src/dash-table/syntax-tree/index.ts +++ b/src/dash-table/syntax-tree/index.ts @@ -11,7 +11,7 @@ export const getMultiColumnQueryString = ( asts: SingleColumnSyntaxTree[] ) => R.map( ast => ast.toQueryString(), - R.filter(ast => ast && ast.isValid, asts) + R.filter(ast => ast && ast.isValid, asts) ).join(' && '); export const getSingleColumnMap = (ast: MultiColumnsSyntaxTree) => { From 0e73b9a6fd416bd27274f369d8d96df481e01184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 17 Apr 2019 15:09:49 -0400 Subject: [PATCH 03/32] type awareness in single column query --- src/dash-table/components/FilterFactory.tsx | 29 ++++++++------- .../syntax-tree/SingleColumnSyntaxTree.ts | 36 ++++++++++++------- src/dash-table/syntax-tree/index.ts | 19 +++++++--- src/dash-table/syntax-tree/lexicon/column.ts | 28 +++++++++++---- 4 files changed, 74 insertions(+), 38 deletions(-) diff --git a/src/dash-table/components/FilterFactory.tsx b/src/dash-table/components/FilterFactory.tsx index 20695e2e6..31bc960cb 100644 --- a/src/dash-table/components/FilterFactory.tsx +++ b/src/dash-table/components/FilterFactory.tsx @@ -13,7 +13,6 @@ import derivedHeaderOperations from 'dash-table/derived/header/operations'; import { derivedRelevantFilterStyles } from 'dash-table/derived/style'; import { BasicFilters, Cells, Style } from 'dash-table/derived/style/props'; import { MultiColumnsSyntaxTree, SingleColumnSyntaxTree, getMultiColumnQueryString, getSingleColumnMap } from 'dash-table/syntax-tree'; -import SyntaxTree from 'core/syntax-tree'; type SetFilter = (filter: string, rawFilter: string) => void; @@ -49,14 +48,14 @@ export default class FilterFactory { } - private onChange = (columnId: ColumnId, setFilter: SetFilter, ev: any) => { - Logger.debug('Filter -- onChange', columnId, ev.target.value && ev.target.value.trim()); + private onChange = (column: IVisibleColumn, setFilter: SetFilter, ev: any) => { + Logger.debug('Filter -- onChange', column.id, ev.target.value && ev.target.value.trim()); const value = ev.target.value.trim(); - const safeColumnId = columnId.toString(); + const safeColumnId = column.id.toString(); if (value && value.length) { - this.ops.set(safeColumnId, new SingleColumnSyntaxTree(safeColumnId, value)); + this.ops.set(safeColumnId, new SingleColumnSyntaxTree(value, { id: column.id, type: column.type })); } else { this.ops.delete(safeColumnId); } @@ -66,26 +65,26 @@ export default class FilterFactory { const rawGlobalFilter = R.map( ast => ast.query || '', - R.filter(ast => Boolean(ast), asts) + R.filter(ast => Boolean(ast), asts) ).join(' && '); setFilter(globalFilter, rawGlobalFilter); } - private getEventHandler = (fn: Function, columnId: ColumnId, setFilter: SetFilter): any => { + private getEventHandler = (fn: Function, column: IVisibleColumn, setFilter: SetFilter): any => { const fnHandler = (this.handlers.get(fn) || this.handlers.set(fn, new Map()).get(fn)); - const columnIdHandler = (fnHandler.get(columnId) || fnHandler.set(columnId, new Map()).get(columnId)); + const columnIdHandler = (fnHandler.get(column.id) || fnHandler.set(column.id, new Map()).get(column.id)); return ( columnIdHandler.get(setFilter) || - (columnIdHandler.set(setFilter, fn.bind(this, columnId, setFilter)).get(setFilter)) + (columnIdHandler.set(setFilter, fn.bind(this, column, setFilter)).get(setFilter)) ); } - private updateOps = memoizeOne((query: string) => { + private updateOps = memoizeOne((query: string, columns: IVisibleColumn[]) => { const multiQuery = new MultiColumnsSyntaxTree(query); - const newOps = getSingleColumnMap(multiQuery); + const newOps = getSingleColumnMap(multiQuery, columns); if (!newOps) { return; } @@ -110,7 +109,7 @@ export default class FilterFactory { }); private filter = memoizerCache<[ColumnId, number]>()(( - column: ColumnId, + column: IVisibleColumn, index: number, ast: SingleColumnSyntaxTree | undefined, setFilter: SetFilter @@ -118,7 +117,7 @@ export default class FilterFactory { return ((R.map)((column, index) => { return this.filter.get(column.id, index)( - column.id, + column, index, this.ops.get(column.id.toString()), setFilter diff --git a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts index 6ed70aa9f..5f511ed71 100644 --- a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts +++ b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts @@ -2,41 +2,46 @@ import SyntaxTree from 'core/syntax-tree'; import { ILexemeResult, ILexerResult } from 'core/syntax-tree/lexer'; import { LexemeType, boundLexeme } from 'core/syntax-tree/lexicon'; -import { ColumnId } from 'dash-table/components/Table/props'; +import { ColumnType } from 'dash-table/components/Table/props'; import operand from './lexeme/operand'; -import { equal } from './lexeme/relational'; +import { equal, RelationalOperator } from './lexeme/relational'; -import columnLexicon from './lexicon/column'; +import columnLexicon, { ISingleColumnConfig } from './lexicon/column'; -function isBinary(lexemes: ILexemeResult[]) { +function isBinary(lexemes: ILexemeResult[]) { return lexemes.length === 2; } -function isExpression(lexemes: ILexemeResult[]) { +function isExpression(lexemes: ILexemeResult[]) { return lexemes.length === 1 && lexemes[0].lexeme.type === LexemeType.Expression; } -function isUnary(lexemes: ILexemeResult[]) { +function isUnary(lexemes: ILexemeResult[]) { return lexemes.length === 1 && lexemes[0].lexeme.type === LexemeType.UnaryOperator; } -export function modifyLex(key: ColumnId, res: ILexerResult) { +export function modifyLex(config: ISingleColumnConfig, res: ILexerResult) { if (!res.valid) { return res; } if (isBinary(res.lexemes) || isUnary(res.lexemes)) { res.lexemes = [ - { lexeme: boundLexeme(operand), value: `{${key}}` }, + { lexeme: boundLexeme(operand), value: `{${config.id}}` }, ...res.lexemes ]; } else if (isExpression(res.lexemes)) { res.lexemes = [ - { lexeme: boundLexeme(operand), value: `{${key}}` }, - { lexeme: boundLexeme(equal), value: 'eq' }, + { lexeme: boundLexeme(operand), value: `{${config.id}}` }, + { + lexeme: boundLexeme(equal), + value: config.type === ColumnType.Text ? + RelationalOperator.Contains : + RelationalOperator.Equal + }, ...res.lexemes ]; } @@ -44,8 +49,13 @@ export function modifyLex(key: ColumnId, res: ILexerResult) { return res; } -export default class SingleColumnSyntaxTree extends SyntaxTree { - constructor(key: ColumnId, query: string) { - super(columnLexicon, query, undefined, modifyLex.bind(undefined, key)); +export default class SingleColumnSyntaxTree extends SyntaxTree { + constructor(query: string, config: ISingleColumnConfig) { + super( + columnLexicon, + query, + config, + modifyLex.bind(undefined, config) + ); } } \ No newline at end of file diff --git a/src/dash-table/syntax-tree/index.ts b/src/dash-table/syntax-tree/index.ts index ada1e35a3..ba67f028e 100644 --- a/src/dash-table/syntax-tree/index.ts +++ b/src/dash-table/syntax-tree/index.ts @@ -6,6 +6,7 @@ import MultiColumnsSyntaxTree from './MultiColumnsSyntaxTree'; import QuerySyntaxTree from './QuerySyntaxTree'; import SingleColumnSyntaxTree from './SingleColumnSyntaxTree'; import { RelationalOperator } from './lexeme/relational'; +import { IVisibleColumn } from 'dash-table/components/Table/props'; export const getMultiColumnQueryString = ( asts: SingleColumnSyntaxTree[] @@ -14,7 +15,10 @@ export const getMultiColumnQueryString = ( R.filter(ast => ast && ast.isValid, asts) ).join(' && '); -export const getSingleColumnMap = (ast: MultiColumnsSyntaxTree) => { +export const getSingleColumnMap = ( + ast: MultiColumnsSyntaxTree, + columns: IVisibleColumn[] +) => { if (!ast.isValid) { return; } @@ -29,14 +33,21 @@ export const getSingleColumnMap = (ast: MultiColumnsSyntaxTree) => { R.forEach(s => { if (s.lexeme.type === LexemeType.UnaryOperator && s.left) { const sanitizedColumnId = s.left.lexeme.present ? s.left.lexeme.present(s.left) : s.left.value; - map.set(sanitizedColumnId, new SingleColumnSyntaxTree(sanitizedColumnId, s.value)); + + const column = R.find(c => c.id === sanitizedColumnId, columns); + const config = { id: sanitizedColumnId, type: column && column.type }; + + map.set(sanitizedColumnId, new SingleColumnSyntaxTree(s.value, config)); } else if (s.lexeme.type === LexemeType.RelationalOperator && s.left && s.right) { const sanitizedColumnId = s.left.lexeme.present ? s.left.lexeme.present(s.left) : s.left.value; + const column = R.find(c => c.id === sanitizedColumnId, columns); + const config = { id: sanitizedColumnId, type: column && column.type }; + if (s.lexeme.present && s.lexeme.present(s) === RelationalOperator.Equal) { - map.set(sanitizedColumnId, new SingleColumnSyntaxTree(sanitizedColumnId, `${s.right.value}`)); + map.set(sanitizedColumnId, new SingleColumnSyntaxTree(`${s.right.value}`, config)); } else { - map.set(sanitizedColumnId, new SingleColumnSyntaxTree(sanitizedColumnId, `${s.value} ${s.right.value}`)); + map.set(sanitizedColumnId, new SingleColumnSyntaxTree(`${s.value} ${s.right.value}`, config)); } } }, statements); diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index b6e42ec6d..02e888d43 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -27,10 +27,17 @@ import { import { LexemeType, ILexeme } from 'core/syntax-tree/lexicon'; import { ILexemeResult } from 'core/syntax-tree/lexer'; +import { ColumnId, ColumnType } from 'dash-table/components/Table/props'; -const lexicon: ILexeme[] = [ - ...[contains, - equal, +export interface ISingleColumnConfig { + id: ColumnId; + type: ColumnType | undefined; +} + +type LexemeResult = ILexemeResult; + +const lexicon: ILexeme[] = [ + ...[equal, greaterOrEqual, greaterThan, lessOrEqual, @@ -39,8 +46,17 @@ const lexicon: ILexeme[] = [ ].map(op => ({ ...op, terminal: false, - if: (_lexs: ILexemeResult[], previous: ILexemeResult) => !previous + if: (_lexs: LexemeResult[], previous: LexemeResult) => !previous })), + { + ...contains, + terminal: false, + if: ( + _lexs: LexemeResult[], + previous: LexemeResult, + config: ISingleColumnConfig + ) => !previous && config.type === ColumnType.Text + }, ...[isBool, isEven, isNil, @@ -51,7 +67,7 @@ const lexicon: ILexeme[] = [ isStr ].map(op => ({ ...op, - if: (_lexs: ILexemeResult[], previous: ILexemeResult) => !previous, + if: (_lexs: LexemeResult[], previous: LexemeResult) => !previous, terminal: true })), ...[ @@ -60,7 +76,7 @@ const lexicon: ILexeme[] = [ valueExpression ].map(exp => ({ ...exp, - if: (_lexs: ILexemeResult[], previous: ILexemeResult) => + if: (_lexs: LexemeResult[], previous: LexemeResult) => !previous || R.contains( previous.lexeme.type, [LexemeType.RelationalOperator] From 92c744c85e14e2f1bcb9d56565eaa54278fb0b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 17 Apr 2019 15:33:10 -0400 Subject: [PATCH 04/32] add standalone and unit tests for `contains` vs `eq` by-type behavior --- .../tests/standalone/filtering_test.ts | 18 +++++ .../unit/single_column_syntactic_tree_test.ts | 77 ++++++++++++++++--- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/tests/cypress/tests/standalone/filtering_test.ts b/tests/cypress/tests/standalone/filtering_test.ts index fe2b2c856..7417860da 100644 --- a/tests/cypress/tests/standalone/filtering_test.ts +++ b/tests/cypress/tests/standalone/filtering_test.ts @@ -95,6 +95,24 @@ describe('filter', () => { DashTable.getFilterById('eee').should('have.class', 'invalid'); }); + it('filters `Text` columns with `contains` without operator', () => { + DashTable.getFilterById('bbb').click(); + DOM.focused.type('Tr'); + DashTable.getFilterById('ccc').click(); + + DashTable.getFilterById('bbb').within(() => cy.get('input').should('have.value', 'Tr')); + DashTable.getCellById(0, 'bbb-readonly').within(() => cy.get('.dash-cell-value').should('have.html', 'Tropical Beaches')); + }); + + it('filters `Numeric` columns with `equal` without operator', () => { + DashTable.getFilterById('ccc').click(); + DOM.focused.type('100'); + DashTable.getFilterById('bbb').click(); + + DashTable.getFilterById('ccc').within(() => cy.get('input').should('have.value', '100')); + DashTable.getCellById(0, 'ccc').within(() => cy.get('.dash-cell-value').should('have.html', '100')); + }); + it('reset updates results and filter fields', () => { let cell_0; let cell_1; diff --git a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts index dacdcc140..feb2fa5f6 100644 --- a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts +++ b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts @@ -1,33 +1,54 @@ import { SingleColumnSyntaxTree } from 'dash-table/syntax-tree'; +import { ColumnType } from 'dash-table/components/Table/props'; + +const COLUMN_ANY = { + id: 'a', + type: ColumnType.Any +}; + +const COLUMN_NUMERIC = { + id: 'a', + type: ColumnType.Numeric +}; + +const COLUMN_TEXT = { + id: 'a', + type: ColumnType.Text +}; + +const COLUMN_UNDEFINED = { + id: 'a', + type: undefined +}; describe('Single Column Syntax Tree', () => { it('cannot have operand', () => { - const tree = new SingleColumnSyntaxTree('a', '{a} <= 1'); + const tree = new SingleColumnSyntaxTree('{a} <= 1', COLUMN_UNDEFINED); expect(tree.isValid).to.equal(false); }); it('cannot have binary dangle', () => { - const tree = new SingleColumnSyntaxTree('a', '<='); + const tree = new SingleColumnSyntaxTree('<=', COLUMN_UNDEFINED); expect(tree.isValid).to.equal(false); }); it('cannot be unary + expression', () => { - const tree = new SingleColumnSyntaxTree('a', 'is prime "a"'); + const tree = new SingleColumnSyntaxTree('is prime "a"', COLUMN_UNDEFINED); expect(tree.isValid).to.equal(false); }); it('can be empty', () => { - const tree = new SingleColumnSyntaxTree('a', ''); + const tree = new SingleColumnSyntaxTree('', COLUMN_UNDEFINED); expect(tree.isValid).to.equal(true); expect(tree.evaluate({ a: 0 })).to.equal(true); }); it('can be binary + expression', () => { - const tree = new SingleColumnSyntaxTree('a', '<= 1'); + const tree = new SingleColumnSyntaxTree('<= 1', COLUMN_UNDEFINED); expect(tree.isValid).to.equal(true); expect(tree.evaluate({ a: 0 })).to.equal(true); @@ -37,7 +58,7 @@ describe('Single Column Syntax Tree', () => { }); it('can be unary', () => { - const tree = new SingleColumnSyntaxTree('a', 'is prime'); + const tree = new SingleColumnSyntaxTree('is prime', COLUMN_UNDEFINED); expect(tree.isValid).to.equal(true); expect(tree.evaluate({ a: 5 })).to.equal(true); @@ -46,13 +67,51 @@ describe('Single Column Syntax Tree', () => { expect(tree.toQueryString()).to.equal('{a} is prime'); }); - it('can be expression', () => { - const tree = new SingleColumnSyntaxTree('a', '1'); + it('can be expression with undefined column type', () => { + const tree = new SingleColumnSyntaxTree('1', COLUMN_UNDEFINED); + + expect(tree.isValid).to.equal(true); + expect(tree.evaluate({ a: 1 })).to.equal(true); + expect(tree.evaluate({ a: 2 })).to.equal(false); + + expect(tree.toQueryString()).to.equal('{a} = 1'); + }); + + it('can be expression with numeric column type', () => { + const tree = new SingleColumnSyntaxTree('1', COLUMN_NUMERIC); expect(tree.isValid).to.equal(true); expect(tree.evaluate({ a: 1 })).to.equal(true); expect(tree.evaluate({ a: 2 })).to.equal(false); - expect(tree.toQueryString()).to.equal('{a} eq 1'); + expect(tree.toQueryString()).to.equal('{a} = 1'); + }); + + it('`undefined` column type cannot use `contains`', () => { + const tree = new SingleColumnSyntaxTree('contains 1', COLUMN_UNDEFINED); + + expect(tree.isValid).to.equal(false); + }); + + it('`any` column type cannot use `contains`', () => { + const tree = new SingleColumnSyntaxTree('contains 1', COLUMN_ANY); + + expect(tree.isValid).to.equal(false); + }); + + it('`numeric` column type cannot use `contains`', () => { + const tree = new SingleColumnSyntaxTree('contains 1', COLUMN_NUMERIC); + + expect(tree.isValid).to.equal(false); + }); + + it('can be expression with text column type', () => { + const tree = new SingleColumnSyntaxTree('"1"', COLUMN_TEXT); + + expect(tree.isValid).to.equal(true); + expect(tree.evaluate({ a: '1' })).to.equal(true); + expect(tree.evaluate({ a: '2' })).to.equal(false); + + expect(tree.toQueryString()).to.equal('{a} contains "1"'); }); }); \ No newline at end of file From 4406df464bb02df273eba649c2c34b54495e5504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 17 Apr 2019 15:58:01 -0400 Subject: [PATCH 05/32] clean up --- src/dash-table/syntax-tree/lexicon/query.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dash-table/syntax-tree/lexicon/query.ts b/src/dash-table/syntax-tree/lexicon/query.ts index 9a8fca8ea..8b23c1b63 100644 --- a/src/dash-table/syntax-tree/lexicon/query.ts +++ b/src/dash-table/syntax-tree/lexicon/query.ts @@ -67,7 +67,7 @@ const ifOperator = (_: ILexemeResult[], previous: ILexemeResult) => [LexemeType.Operand] ); -const untypedQueryLexicon = [ +const lexicon: ILexeme[] = [ { ...and, if: ifLogicalOperator, @@ -165,6 +165,4 @@ const untypedQueryLexicon = [ })) ]; -const lexicon: ILexeme[] = untypedQueryLexicon; - export default lexicon; \ No newline at end of file From 1089f8626dca6d333fd02bf548aee1308711c8f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Tue, 23 Apr 2019 16:38:39 -0400 Subject: [PATCH 06/32] - likeDate relational operator - generic evaluate override (uses config) --- src/core/syntax-tree/index.ts | 6 ++++++ src/core/syntax-tree/lexicon.ts | 10 +++++++--- src/dash-table/syntax-tree/lexeme/relational.ts | 16 ++++++++++++++++ src/dash-table/syntax-tree/lexicon/column.ts | 12 ++---------- src/dash-table/type/date.ts | 2 +- 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/core/syntax-tree/index.ts b/src/core/syntax-tree/index.ts index 4ad325a39..442217333 100644 --- a/src/core/syntax-tree/index.ts +++ b/src/core/syntax-tree/index.ts @@ -57,6 +57,12 @@ export default class SyntaxTree { public readonly config: TConfig, postProcessor: (res: ILexerResult) => ILexerResult = res => res ) { + this.lexicon = R.map(lexeme => R.merge(lexeme, { + evaluate: lexeme.boundedEvaluate ? + lexeme.boundedEvaluate.bind(undefined, config) : + lexeme.evaluate + }), lexicon); + this.lexerResult = postProcessor(configurableLexer(this.lexicon, this.query, this.config)); this.syntaxerResult = syntaxer(this.lexerResult); } diff --git a/src/core/syntax-tree/lexicon.ts b/src/core/syntax-tree/lexicon.ts index 96b54138d..6896f5884 100644 --- a/src/core/syntax-tree/lexicon.ts +++ b/src/core/syntax-tree/lexicon.ts @@ -24,17 +24,21 @@ export interface IUnboundedLexeme { syntaxer?: (lexs: any[], pivot: any, pivotIndex: number) => any; } -type Terminal = boolean | +type Evaluate = + ((config: TConfig, target: any, tree: ISyntaxTree) => boolean); + +type If = (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean) | ((lexemes: ILexemeResult[], previous: ILexemeResult, config: TConfig) => boolean); -type If = (string | undefined)[] | +type Terminal = boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean) | ((lexemes: ILexemeResult[], previous: ILexemeResult, config: TConfig) => boolean); export interface ILexeme extends IUnboundedLexeme { - terminal: Terminal; + boundedEvaluate?: Evaluate; if: If; + terminal: Terminal; } export function boundLexeme(lexeme: IUnboundedLexeme) { diff --git a/src/dash-table/syntax-tree/lexeme/relational.ts b/src/dash-table/syntax-tree/lexeme/relational.ts index df6fb9a78..ac89d4e86 100644 --- a/src/dash-table/syntax-tree/lexeme/relational.ts +++ b/src/dash-table/syntax-tree/lexeme/relational.ts @@ -3,6 +3,7 @@ import * as R from 'ramda'; import Logger from 'core/Logger'; import { LexemeType, IUnboundedLexeme } from 'core/syntax-tree/lexicon'; import { ISyntaxTree } from 'core/syntax-tree/syntaxer'; +import { normalizeDate } from 'dash-table/type/date'; function evaluator( target: any, @@ -36,6 +37,7 @@ export enum RelationalOperator { GreaterThan = '>', LessOrEqual = '<=', LessThan = '<', + LikeDate = 'like_date', NotEqual = '!=' } @@ -73,6 +75,20 @@ export const greaterThan: IUnboundedLexeme = R.merge({ regexp: /^(>|gt)/i }, LEXEME_BASE); +export const likeDate: IUnboundedLexeme = R.merge({ + evaluate: relationalEvaluator(([op, exp]) => { + const normalizedOp = normalizeDate(op); + const normalizedExp = normalizeDate(exp); + + return normalizedOp !== null && + normalizedExp !== null && + // IE11 does not support `startsWith` + normalizedOp.indexOf(normalizedExp) === 0; + }), + subType: RelationalOperator.LikeDate, + regexp: /^(in)/i +}, LEXEME_BASE); + export const lessOrEqual: IUnboundedLexeme = R.merge({ evaluate: relationalEvaluator(([op, exp]) => op <= exp), subType: RelationalOperator.LessOrEqual, diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index 02e888d43..650afba46 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -37,7 +37,8 @@ export interface ISingleColumnConfig { type LexemeResult = ILexemeResult; const lexicon: ILexeme[] = [ - ...[equal, + ...[contains, + equal, greaterOrEqual, greaterThan, lessOrEqual, @@ -48,15 +49,6 @@ const lexicon: ILexeme[] = [ terminal: false, if: (_lexs: LexemeResult[], previous: LexemeResult) => !previous })), - { - ...contains, - terminal: false, - if: ( - _lexs: LexemeResult[], - previous: LexemeResult, - config: ISingleColumnConfig - ) => !previous && config.type === ColumnType.Text - }, ...[isBool, isEven, isNil, diff --git a/src/dash-table/type/date.ts b/src/dash-table/type/date.ts index 45a820efe..a441da087 100644 --- a/src/dash-table/type/date.ts +++ b/src/dash-table/type/date.ts @@ -13,7 +13,7 @@ const DATETIME_REGEXP = /^\s*(-?\d{4}|\d{2})(-(\d{1,2})(-(\d{1,2})([ Tt]([01]?\d // Please don't use 2-digit years! const YFIRST = new Date().getFullYear() - 70; -export function normalizeDate(value: any, options: IDatetimeColumn | undefined): string | null { +export function normalizeDate(value: any, options?: IDatetimeColumn): string | null { // unlike plotly.js, do not accept year as a number - only strings. if (typeof value !== 'string') { return null; From eccf3448da8dafb083928ee1afcda731792c619c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Tue, 23 Apr 2019 20:59:58 -0400 Subject: [PATCH 07/32] contains usable by all column types --- .../tests/unit/single_column_syntactic_tree_test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts index feb2fa5f6..cd3111981 100644 --- a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts +++ b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts @@ -87,22 +87,22 @@ describe('Single Column Syntax Tree', () => { expect(tree.toQueryString()).to.equal('{a} = 1'); }); - it('`undefined` column type cannot use `contains`', () => { + it('`undefined` column type can use `contains`', () => { const tree = new SingleColumnSyntaxTree('contains 1', COLUMN_UNDEFINED); - expect(tree.isValid).to.equal(false); + expect(tree.isValid).to.equal(true); }); - it('`any` column type cannot use `contains`', () => { + it('`any` column type can use `contains`', () => { const tree = new SingleColumnSyntaxTree('contains 1', COLUMN_ANY); - expect(tree.isValid).to.equal(false); + expect(tree.isValid).to.equal(true); }); - it('`numeric` column type cannot use `contains`', () => { + it('`numeric` column type can use `contains`', () => { const tree = new SingleColumnSyntaxTree('contains 1', COLUMN_NUMERIC); - expect(tree.isValid).to.equal(false); + expect(tree.isValid).to.equal(true); }); it('can be expression with text column type', () => { From 6ff91faf26e7402b28bc73393f57c076c489bfa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Tue, 23 Apr 2019 21:50:34 -0400 Subject: [PATCH 08/32] - update single column config - update associated tests & usage --- src/dash-table/components/FilterFactory.tsx | 2 +- .../syntax-tree/SingleColumnSyntaxTree.ts | 6 ++-- src/dash-table/syntax-tree/index.ts | 12 +++++-- src/dash-table/syntax-tree/lexicon/column.ts | 7 ++-- .../unit/single_column_syntactic_tree_test.ts | 33 ++++++++++++------- 5 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/dash-table/components/FilterFactory.tsx b/src/dash-table/components/FilterFactory.tsx index 31bc960cb..6e7ec9813 100644 --- a/src/dash-table/components/FilterFactory.tsx +++ b/src/dash-table/components/FilterFactory.tsx @@ -55,7 +55,7 @@ export default class FilterFactory { const safeColumnId = column.id.toString(); if (value && value.length) { - this.ops.set(safeColumnId, new SingleColumnSyntaxTree(value, { id: column.id, type: column.type })); + this.ops.set(safeColumnId, new SingleColumnSyntaxTree(value, { column })); } else { this.ops.delete(safeColumnId); } diff --git a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts index 5f511ed71..4312b541b 100644 --- a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts +++ b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts @@ -30,15 +30,15 @@ export function modifyLex(config: ISingleColumnConfig, res: ILexerResult c.id === sanitizedColumnId, columns); - const config = { id: sanitizedColumnId, type: column && column.type }; + if (!column) { + throw new Error(`column ${sanitizedColumnId} not found`); + } + + const config = { column }; map.set(sanitizedColumnId, new SingleColumnSyntaxTree(s.value, config)); } else if (s.lexeme.type === LexemeType.RelationalOperator && s.left && s.right) { const sanitizedColumnId = s.left.lexeme.present ? s.left.lexeme.present(s.left) : s.left.value; const column = R.find(c => c.id === sanitizedColumnId, columns); - const config = { id: sanitizedColumnId, type: column && column.type }; + if (!column) { + throw new Error(`column ${sanitizedColumnId} not found`); + } + + const config = { column }; if (s.lexeme.present && s.lexeme.present(s) === RelationalOperator.Equal) { map.set(sanitizedColumnId, new SingleColumnSyntaxTree(`${s.right.value}`, config)); diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index 650afba46..a5eaf8523 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -27,11 +27,12 @@ import { import { LexemeType, ILexeme } from 'core/syntax-tree/lexicon'; import { ILexemeResult } from 'core/syntax-tree/lexer'; -import { ColumnId, ColumnType } from 'dash-table/components/Table/props'; +import { ColumnId, IVisibleColumn } from 'dash-table/components/Table/props'; export interface ISingleColumnConfig { - id: ColumnId; - type: ColumnType | undefined; + column: Partial & { + id: ColumnId + }; } type LexemeResult = ILexemeResult; diff --git a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts index cd3111981..6878a8e72 100644 --- a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts +++ b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts @@ -1,24 +1,33 @@ import { SingleColumnSyntaxTree } from 'dash-table/syntax-tree'; import { ColumnType } from 'dash-table/components/Table/props'; +import { ISingleColumnConfig } from 'dash-table/syntax-tree/lexicon/column'; -const COLUMN_ANY = { - id: 'a', - type: ColumnType.Any +const COLUMN_ANY: ISingleColumnConfig = { + column: { + id: 'a', + type: ColumnType.Any + } }; -const COLUMN_NUMERIC = { - id: 'a', - type: ColumnType.Numeric +const COLUMN_NUMERIC: ISingleColumnConfig = { + column: { + id: 'a', + type: ColumnType.Numeric + } }; -const COLUMN_TEXT = { - id: 'a', - type: ColumnType.Text +const COLUMN_TEXT: ISingleColumnConfig = { + column: { + id: 'a', + type: ColumnType.Text + } }; -const COLUMN_UNDEFINED = { - id: 'a', - type: undefined +const COLUMN_UNDEFINED: ISingleColumnConfig = { + column: { + id: 'a', + type: undefined + } }; describe('Single Column Syntax Tree', () => { From b624c3709e2bd7f13905f51b05af0d8ad8e168f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 09:27:56 -0400 Subject: [PATCH 09/32] fix e2e test (legacy impl) --- src/dash-table/syntax-tree/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dash-table/syntax-tree/index.ts b/src/dash-table/syntax-tree/index.ts index 7ad32eb47..a3654805f 100644 --- a/src/dash-table/syntax-tree/index.ts +++ b/src/dash-table/syntax-tree/index.ts @@ -34,7 +34,7 @@ export const getSingleColumnMap = ( if (s.lexeme.type === LexemeType.UnaryOperator && s.left) { const sanitizedColumnId = s.left.lexeme.present ? s.left.lexeme.present(s.left) : s.left.value; - const column = R.find(c => c.id === sanitizedColumnId, columns); + const column = R.find(c => c.id.toString() === sanitizedColumnId, columns); if (!column) { throw new Error(`column ${sanitizedColumnId} not found`); } @@ -45,7 +45,7 @@ export const getSingleColumnMap = ( } else if (s.lexeme.type === LexemeType.RelationalOperator && s.left && s.right) { const sanitizedColumnId = s.left.lexeme.present ? s.left.lexeme.present(s.left) : s.left.value; - const column = R.find(c => c.id === sanitizedColumnId, columns); + const column = R.find(c => c.id.toString() === sanitizedColumnId, columns); if (!column) { throw new Error(`column ${sanitizedColumnId} not found`); } From aeac7fc38c8f1cc8cbf5c83c0232f0745f7ca09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 10:36:39 -0400 Subject: [PATCH 10/32] update ci test breakdown --- .circleci/config.yml | 64 +++++++++++++++++++++++++++++++++++++++++--- package.json | 16 ++++++----- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 619292432..cce8c6f08 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2 jobs: - "test": + "server-test": docker: - image: circleci/python:3.6.7-node-browsers - image: cypress/base:10 @@ -44,7 +44,63 @@ jobs: name: Run tests command: | . venv/bin/activate - npm run test + npm run test.server + + + "standalone-test": + docker: + - image: circleci/python:3.6.7-node-browsers + - image: cypress/base:10 + + steps: + - checkout + - restore_cache: + key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }} + - run: + name: Install npm packages + command: npm install + - run: + name: Cypress Install + command: | + $(npm bin)/cypress install + + - save_cache: + key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }} + paths: + - node_modules + - /home/circleci/.cache/Cypress + + - run: + name: Run tests + command: npm run test.standalone + + + "unit-test": + docker: + - image: circleci/python:3.6.7-node-browsers + - image: cypress/base:10 + + steps: + - checkout + - restore_cache: + key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }} + - run: + name: Install npm packages + command: npm install + - run: + name: Cypress Install + command: | + $(npm bin)/cypress install + + - save_cache: + key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }} + paths: + - node_modules + - /home/circleci/.cache/Cypress + + - run: + name: Run tests + command: npm run test.unit "visual-test": @@ -170,5 +226,7 @@ workflows: jobs: - "python-3.6" - "node" - - "test" + - "server-test" + - "standalone-test" + - "unit-test" - "visual-test" diff --git a/package.json b/package.json index 49601623f..9a40161ad 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "main": "dash_table/bundle.js", "scripts": { "preprivate::opentests": "run-s private::wait*", - "preprivate::runtests": "run-s private::wait*", + "preprivate::test.server": "run-s private::wait_dash*", + "preprivate::test.standalone": "run-s private::wait_js*", "private::build": "node --max_old_space_size=4096 node_modules/webpack/bin/webpack --display-reasons --bail", "private::build:js": "run-s \"private::build -- --mode production\"", "private::build:js-dev": "run-s \"private::build -- --mode development\"", @@ -26,15 +27,16 @@ "private::wait_dash8083": "wait-on http://localhost:8083", "private::wait_js": "wait-on http://localhost:8080", "private::opentests": "cypress open", - "private::runtests:python": "python -m unittest tests/unit/format_test.py", - "private::runtests:unit": "cypress run --browser chrome --spec 'tests/cypress/tests/unit/**/*'", - "private::runtests:standalone": "cypress run --browser chrome --spec 'tests/cypress/tests/standalone/**/*'", - "private::runtests:server": "cypress run --browser chrome --spec 'tests/cypress/tests/server/**/*'", - "private::runtests": "run-s private::runtests:python private::runtests:unit private::runtests:standalone private::runtests:server", + "private::test.python": "python -m unittest tests/unit/format_test.py", + "private::test.unit": "cypress run --browser chrome --spec 'tests/cypress/tests/unit/**/*'", + "private::test.server": "cypress run --browser chrome --spec 'tests/cypress/tests/server/**/*'", + "private::test.standalone": "cypress run --browser chrome --spec 'tests/cypress/tests/standalone/**/*'", "build.watch": "webpack-dev-server --content-base dash_table --mode development", "build": "run-s private::build:js private::build:py", "lint": "run-s private::lint:*", - "test": "run-p --race private::host* private::runtests", + "test.server": "run-p --race private::host* private::test.server", + "test.standalone": "run-p --race private::host_js private::test.standalone", + "test.unit": "run-s private::test.python private::test.unit", "test.visual": "build-storybook && percy-storybook", "test.visual-local": "build-storybook", "test.watch": "run-p --race \"private::build:js-test-watch\" --race private::host* private::opentests" From 32ac750ffcb26665d3fba2ec1731f4c5f37f9abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 10:42:49 -0400 Subject: [PATCH 11/32] py env for py tests --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cce8c6f08..380bba577 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,6 +98,14 @@ jobs: - node_modules - /home/circleci/.cache/Cypress + - run: + name: Install requirements + command: | + sudo pip install --upgrade virtualenv + python -m venv venv || virtualenv venv + . venv/bin/activate + pip install -r requirements.txt --quiet + - run: name: Run tests command: npm run test.unit From eb9ece4a557a84a6abec98ab9a138b6c81a6c32a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 10:48:34 -0400 Subject: [PATCH 12/32] unit tests venv --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 380bba577..4a9199b90 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,7 +108,9 @@ jobs: - run: name: Run tests - command: npm run test.unit + command: | + . venv/bin/activate + npm run test.unit "visual-test": From 2ec2812b169d23e8de0026f676c9a99ec1ccc54f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 11:10:16 -0400 Subject: [PATCH 13/32] update standalone tests --- package.json | 2 +- .../syntax-tree/lexeme/relational.ts | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 9a40161ad..1f5ce4443 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "preprivate::opentests": "run-s private::wait*", "preprivate::test.server": "run-s private::wait_dash*", - "preprivate::test.standalone": "run-s private::wait_js*", + "preprivate::test.standalone": "run-s private::build:js-test private::wait_js", "private::build": "node --max_old_space_size=4096 node_modules/webpack/bin/webpack --display-reasons --bail", "private::build:js": "run-s \"private::build -- --mode production\"", "private::build:js-dev": "run-s \"private::build -- --mode development\"", diff --git a/src/dash-table/syntax-tree/lexeme/relational.ts b/src/dash-table/syntax-tree/lexeme/relational.ts index ac89d4e86..6f0e1b8bb 100644 --- a/src/dash-table/syntax-tree/lexeme/relational.ts +++ b/src/dash-table/syntax-tree/lexeme/relational.ts @@ -1,3 +1,4 @@ +import isNumeric from 'fast-isnumeric'; import * as R from 'ramda'; import Logger from 'core/Logger'; @@ -49,16 +50,22 @@ const LEXEME_BASE = { export const contains: IUnboundedLexeme = R.merge({ evaluate: relationalEvaluator(([op, exp]) => - typeof op === 'string' && - typeof exp === 'string' && - op.indexOf(exp) !== -1 + !R.isNil(op) && + !R.isNil(exp) && + R.type(op.toString) === 'Function' && + R.type(exp.toString) === 'Function' && + op.toString().indexOf(exp.toString()) !== -1 ), subType: RelationalOperator.Contains, regexp: /^(contains)/i }, LEXEME_BASE); export const equal: IUnboundedLexeme = R.merge({ - evaluate: relationalEvaluator(([op, exp]) => op === exp), + evaluate: relationalEvaluator(([op, exp]) => ( + isNumeric(op) && + isNumeric(exp) && + +op === +exp + ) || op === exp), subType: RelationalOperator.Equal, regexp: /^(=|eq)/i }, LEXEME_BASE); @@ -80,8 +87,8 @@ export const likeDate: IUnboundedLexeme = R.merge({ const normalizedOp = normalizeDate(op); const normalizedExp = normalizeDate(exp); - return normalizedOp !== null && - normalizedExp !== null && + return !R.isNil(normalizedOp) && + !R.isNil(normalizedExp) && // IE11 does not support `startsWith` normalizedOp.indexOf(normalizedExp) === 0; }), From d58611cbaa11189697faf2abec98fca85cde3dda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 11:24:11 -0400 Subject: [PATCH 14/32] allow cross-type `eq` comparison --- tests/cypress/tests/unit/query_syntactic_tree_test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cypress/tests/unit/query_syntactic_tree_test.ts b/tests/cypress/tests/unit/query_syntactic_tree_test.ts index 7fb011b0e..10090079b 100644 --- a/tests/cypress/tests/unit/query_syntactic_tree_test.ts +++ b/tests/cypress/tests/unit/query_syntactic_tree_test.ts @@ -244,12 +244,12 @@ describe('Query Syntax Tree', () => { expect(tree.evaluate({ field: 2 })).to.equal(true); }); - it('can compare string to number and return false', () => { + it('can compare string to number', () => { const tree = new QuerySyntaxTree('{a} eq 1'); expect(tree.isValid).to.equal(true); expect(tree.evaluate(data0)).to.equal(false); - expect(tree.evaluate(data1)).to.equal(false); + expect(tree.evaluate(data1)).to.equal(true); expect(tree.evaluate(data2)).to.equal(false); expect(tree.evaluate(data3)).to.equal(false); }); @@ -264,12 +264,12 @@ describe('Query Syntax Tree', () => { expect(tree.evaluate(data3)).to.equal(false); }); - it('can compare string to number and return false', () => { + it('can compare string to number', () => { const tree = new QuerySyntaxTree('{c} eq "1"'); expect(tree.isValid).to.equal(true); expect(tree.evaluate(data0)).to.equal(false); - expect(tree.evaluate(data1)).to.equal(false); + expect(tree.evaluate(data1)).to.equal(true); expect(tree.evaluate(data2)).to.equal(false); expect(tree.evaluate(data3)).to.equal(false); }); From 17482eee859c9462178399da0376065198875db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 14:36:54 -0400 Subject: [PATCH 15/32] - add 'in' (like date) operator - add tests for 'in' -- all query flavors --- package.json | 3 +- .../syntax-tree/lexeme/relational.ts | 25 ++-- src/dash-table/syntax-tree/lexicon/column.ts | 2 + .../syntax-tree/lexicon/columnMulti.ts | 2 + src/dash-table/syntax-tree/lexicon/query.ts | 2 + src/dash-table/type/date.ts | 10 +- .../tests/unit/dash_table_queries_test.ts | 109 ++++++++++++++++++ .../tests/unit/query_syntactic_tree_test.ts | 29 ++--- 8 files changed, 147 insertions(+), 35 deletions(-) create mode 100644 tests/cypress/tests/unit/dash_table_queries_test.ts diff --git a/package.json b/package.json index 1f5ce4443..e2f278cb7 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "scripts": { "preprivate::opentests": "run-s private::wait*", "preprivate::test.server": "run-s private::wait_dash*", - "preprivate::test.standalone": "run-s private::build:js-test private::wait_js", + "preprivate::test.standalone": "run-s private::wait_js", + "pretest.standalone": "run-s private::build:js-test", "private::build": "node --max_old_space_size=4096 node_modules/webpack/bin/webpack --display-reasons --bail", "private::build:js": "run-s \"private::build -- --mode production\"", "private::build:js-dev": "run-s \"private::build -- --mode development\"", diff --git a/src/dash-table/syntax-tree/lexeme/relational.ts b/src/dash-table/syntax-tree/lexeme/relational.ts index 6f0e1b8bb..dc344ad50 100644 --- a/src/dash-table/syntax-tree/lexeme/relational.ts +++ b/src/dash-table/syntax-tree/lexeme/relational.ts @@ -5,6 +5,7 @@ import Logger from 'core/Logger'; import { LexemeType, IUnboundedLexeme } from 'core/syntax-tree/lexicon'; import { ISyntaxTree } from 'core/syntax-tree/syntaxer'; import { normalizeDate } from 'dash-table/type/date'; +import { IDateValidation } from 'dash-table/components/Table/props'; function evaluator( target: any, @@ -38,7 +39,7 @@ export enum RelationalOperator { GreaterThan = '>', LessOrEqual = '<=', LessThan = '<', - LikeDate = 'like_date', + LikeDate = 'in', NotEqual = '!=' } @@ -49,13 +50,13 @@ const LEXEME_BASE = { }; export const contains: IUnboundedLexeme = R.merge({ - evaluate: relationalEvaluator(([op, exp]) => - !R.isNil(op) && - !R.isNil(exp) && - R.type(op.toString) === 'Function' && - R.type(exp.toString) === 'Function' && - op.toString().indexOf(exp.toString()) !== -1 - ), + evaluate: relationalEvaluator(([op, exp]) => { + if (R.type(exp) !== 'String') { + return false; + } + + return !R.isNil(op) && R.toString(op).indexOf(exp) !== -1; + }), subType: RelationalOperator.Contains, regexp: /^(contains)/i }, LEXEME_BASE); @@ -82,10 +83,14 @@ export const greaterThan: IUnboundedLexeme = R.merge({ regexp: /^(>|gt)/i }, LEXEME_BASE); +const DATE_OPTIONS: IDateValidation = { + allow_YY: true +}; + export const likeDate: IUnboundedLexeme = R.merge({ evaluate: relationalEvaluator(([op, exp]) => { - const normalizedOp = normalizeDate(op); - const normalizedExp = normalizeDate(exp); + const normalizedOp = normalizeDate(op, DATE_OPTIONS); + const normalizedExp = normalizeDate(exp, DATE_OPTIONS); return !R.isNil(normalizedOp) && !R.isNil(normalizedExp) && diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index a5eaf8523..82e78777a 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -12,6 +12,7 @@ import { greaterThan, lessOrEqual, lessThan, + likeDate, notEqual } from '../lexeme/relational'; import { @@ -44,6 +45,7 @@ const lexicon: ILexeme[] = [ greaterThan, lessOrEqual, lessThan, + likeDate, notEqual ].map(op => ({ ...op, diff --git a/src/dash-table/syntax-tree/lexicon/columnMulti.ts b/src/dash-table/syntax-tree/lexicon/columnMulti.ts index a8542b5bc..b379924c6 100644 --- a/src/dash-table/syntax-tree/lexicon/columnMulti.ts +++ b/src/dash-table/syntax-tree/lexicon/columnMulti.ts @@ -16,6 +16,7 @@ import { greaterThan, lessOrEqual, lessThan, + likeDate, notEqual } from '../lexeme/relational'; import { @@ -60,6 +61,7 @@ const lexicon: ILexeme[] = [ greaterThan, lessOrEqual, lessThan, + likeDate, notEqual ].map(op => ({ ...op, diff --git a/src/dash-table/syntax-tree/lexicon/query.ts b/src/dash-table/syntax-tree/lexicon/query.ts index 8b23c1b63..4ddd9ddb2 100644 --- a/src/dash-table/syntax-tree/lexicon/query.ts +++ b/src/dash-table/syntax-tree/lexicon/query.ts @@ -21,6 +21,7 @@ import { greaterThan, lessOrEqual, lessThan, + likeDate, notEqual } from '../lexeme/relational'; import { @@ -123,6 +124,7 @@ const lexicon: ILexeme[] = [ greaterThan, lessOrEqual, lessThan, + likeDate, notEqual ].map(op => ({ ...op, diff --git a/src/dash-table/type/date.ts b/src/dash-table/type/date.ts index a441da087..2f71e5ed0 100644 --- a/src/dash-table/type/date.ts +++ b/src/dash-table/type/date.ts @@ -1,4 +1,4 @@ -import { IDatetimeColumn } from 'dash-table/components/Table/props'; +import { IDatetimeColumn, IDateValidation } from 'dash-table/components/Table/props'; import { reconcileNull } from './null'; import { IReconciliation } from './reconcile'; @@ -13,7 +13,7 @@ const DATETIME_REGEXP = /^\s*(-?\d{4}|\d{2})(-(\d{1,2})(-(\d{1,2})([ Tt]([01]?\d // Please don't use 2-digit years! const YFIRST = new Date().getFullYear() - 70; -export function normalizeDate(value: any, options?: IDatetimeColumn): string | null { +export function normalizeDate(value: any, options?: IDateValidation): string | null { // unlike plotly.js, do not accept year as a number - only strings. if (typeof value !== 'string') { return null; @@ -27,7 +27,7 @@ export function normalizeDate(value: any, options?: IDatetimeColumn): string | n const yearMatch = match[1]; const YY = yearMatch.length === 2; - if (YY && !(options && options.validation && options.validation.allow_YY)) { + if (YY && !(options && options.allow_YY)) { return null; } @@ -81,7 +81,7 @@ export function normalizeDate(value: any, options?: IDatetimeColumn): string | n } export function coerce(value: any, options: IDatetimeColumn | undefined): IReconciliation { - const normalizedDate = normalizeDate(value, options); + const normalizedDate = normalizeDate(value, options && options.validation); return normalizedDate !== null ? { success: true, @@ -91,7 +91,7 @@ export function coerce(value: any, options: IDatetimeColumn | undefined): IRecon } export function validate(value: any, options: IDatetimeColumn | undefined): IReconciliation { - return (typeof value === 'string') && (normalizeDate(value, options) !== null) ? + return (typeof value === 'string') && (normalizeDate(value, options && options.validation) !== null) ? { success: true, value: value.trim() } : reconcileNull(value, options); } diff --git a/tests/cypress/tests/unit/dash_table_queries_test.ts b/tests/cypress/tests/unit/dash_table_queries_test.ts new file mode 100644 index 000000000..ae6d1b447 --- /dev/null +++ b/tests/cypress/tests/unit/dash_table_queries_test.ts @@ -0,0 +1,109 @@ +import * as R from 'ramda'; + +import SyntaxTree from 'core/syntax-tree'; + +import { QuerySyntaxTree, MultiColumnsSyntaxTree, SingleColumnSyntaxTree } from 'dash-table/syntax-tree'; + +export interface ICase { + name: string; + query: string; + target: object; + valid: boolean; + evaluate?: boolean; +} + +export function processCases(getSyntaxer: (query: string) => SyntaxTree, cases: ICase[]) { + R.forEach(c => it(c.name, () => { + const tree = getSyntaxer(c.query); + + expect(tree.isValid).to.equal(c.valid); + if (!c.valid) { + return; + } + + expect(tree.evaluate(c.target)).to.equal(c.evaluate); + }), cases); +} +const getQuerySyntaxTree = (query: string): any => new QuerySyntaxTree(query); +const getMultiColumnSyntaxTree = (query: string): any => new MultiColumnsSyntaxTree(query); +const getSingleColumnSyntaxTree = (query: string): any => new SingleColumnSyntaxTree(query, { + column: { + id: 'a' + } +}); + +describe('Dash Table Queries', () => { + R.forEach(c => { + describe(c.name, () => { + describe('relational operator', () => { + describe('eq', () => { + processCases(c.syntaxer, [ + { name: 'compares "1" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: '1' }, valid: true, evaluate: true }, + { name: 'compares "1" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: '1' }, valid: true, evaluate: true }, + { name: 'compares "1.0" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: '1.0' }, valid: true, evaluate: true }, + { name: 'compares "1.0" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: '1.0' }, valid: true, evaluate: true }, + { name: 'compares "1.1" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: '1.1' }, valid: true, evaluate: false }, + { name: 'compares "1.1" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: '1.1' }, valid: true, evaluate: false }, + { name: 'compares "1" to 0x1', query: `${c.hideOperand ? '' : '{a} '}eq 0x1`, target: { a: '1' }, valid: true, evaluate: true }, + { name: 'compares "1" to "0x1"', query: `${c.hideOperand ? '' : '{a} '}eq "0x1"`, target: { a: '1' }, valid: true, evaluate: true }, + + { name: 'compares 1 to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: 1 }, valid: true, evaluate: true }, + { name: 'compares 1 to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: 1 }, valid: true, evaluate: true }, + { name: 'compares 1.1 to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: 1.1 }, valid: true, evaluate: false }, + { name: 'compares 1.1 to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: 1.1 }, valid: true, evaluate: false }, + { name: 'compares 1 to 0x1', query: `${c.hideOperand ? '' : '{a} '}eq 0x1`, target: { a: 1 }, valid: true, evaluate: true }, + { name: 'compares 1 to "0x1"', query: `${c.hideOperand ? '' : '{a} '}eq "0x1"`, target: { a: 1 }, valid: true, evaluate: true }, + + { name: 'compares "x1" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: 'x1' }, valid: true, evaluate: false }, + { name: 'compares "x1" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: 'x1' }, valid: true, evaluate: false } + ]); + }); + + describe('contains', () => { + processCases(c.syntaxer, [ + { name: 'cannot compare "11" to 1', query: `${c.hideOperand ? '' : '{a} '}contains 1`, target: { a: '11' }, valid: true, evaluate: false }, + { name: 'cannot compare 11 to 1', query: `${c.hideOperand ? '' : '{a} '}contains 1`, target: { a: 11 }, valid: true, evaluate: false }, + { name: 'compares "11" to "1"', query: `${c.hideOperand ? '' : '{a} '}contains "1"`, target: { a: '11' }, valid: true, evaluate: true }, + { name: 'compares 11 to "1"', query: `${c.hideOperand ? '' : '{a} '}contains "1"`, target: { a: 11 }, valid: true, evaluate: true }, + { name: 'compares "1" to "1.0"', query: `${c.hideOperand ? '' : '{a} '}contains "1.0"`, target: { a: '1' }, valid: true, evaluate: false }, + { name: 'compares 1 to "1.0"', query: `${c.hideOperand ? '' : '{a} '}contains "1.0"`, target: { a: 1 }, valid: true, evaluate: false } + ]); + }); + + describe('in', () => { + processCases(c.syntaxer, [ + { name: '0yyy in "0"', query: `${c.hideOperand ? '' : '{a} '}in "0"`, target: { a: '0987' }, valid: true, evaluate: false }, + { name: '0yyy in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}in "0987"`, target: { a: '0987' }, valid: true, evaluate: true }, + { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2006"`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005' }, valid: true, evaluate: true }, + { name: 'yyyy in yyyy', query: `${c.hideOperand ? '' : '{a} '}in 2005`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy-mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, + { name: 'yyyy in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy-mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm in "yyyy-mm"', query: `${c.hideOperand ? '' : '{a} '}in "2005-01"`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx', query: `${c.hideOperand ? '' : '{a} '}in 2005-01-01T10:00:00.000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx000', query: `${c.hideOperand ? '' : '{a} '}in 2005-01-01T10:00:00.000000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx111', query: `${c.hideOperand ? '' : '{a} '}in 2005-01-01T10:00:00.000111`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false } + ]); + }); + }); + }); + }, [ + { name: 'Query Syntax Tree', syntaxer: getQuerySyntaxTree }, + { name: 'Multi Columns Syntax Tree', syntaxer: getMultiColumnSyntaxTree }, + { name: 'Single Column Syntax Tree', syntaxer: getSingleColumnSyntaxTree, hideOperand: true } + ] + ); +}); \ No newline at end of file diff --git a/tests/cypress/tests/unit/query_syntactic_tree_test.ts b/tests/cypress/tests/unit/query_syntactic_tree_test.ts index 10090079b..cdd88771b 100644 --- a/tests/cypress/tests/unit/query_syntactic_tree_test.ts +++ b/tests/cypress/tests/unit/query_syntactic_tree_test.ts @@ -1,7 +1,7 @@ -import * as R from 'ramda'; - import { QuerySyntaxTree } from 'dash-table/syntax-tree'; +import { processCases } from './dash_table_queries_test'; + describe('Query Syntax Tree', () => { const data0 = { a: '0', b: '0', c: 0, d: null, '\\{': 0, 'a.dot': '0.dot', 'a-dot': '0-dot', a_dot: '0_dot', 'a+dot': '0+dot', 'a dot': '0 dot', 'a:dot': '0:dot', '_-6.:+** *@$': '0*dot', '{a:dot}': '0*dot*', '\'""\'': '0\'"dot' }; const data1 = { a: '1', b: '0', c: 1, d: 0, '\\{': 1, 'a.dot': '1.dot', 'a-dot': '1-dot', a_dot: '1_dot', 'a+dot': '1+dot', 'a dot': '1 dot', 'a:dot': '1:dot', '_-6.:+** *@$': '1*dot', '{a:dot}': '1*dot*', '\'""\'': '1\'"dot' }; @@ -9,24 +9,15 @@ describe('Query Syntax Tree', () => { const data3 = { a: '3', b: '1', c: 3, d: false, '\\{': 3, 'a.dot': '3.dot', 'a-dot': '3-dot', a_dot: '3_dot', 'a+dot': '3+dot', 'a dot': '3 dot', 'a:dot': '3:dot', '_-6.:+** *@$': '3*dot', '{a:dot}': '3*dot*', '\'""\'': '3\'"dot' }; describe('special whitespace characters are valid', () => { - const cases = [ - { name: 'suports new line', query: '{a}\neq\n"0"' }, - { name: 'suports carriage return', query: '{a}\req\r"0"' }, - { name: 'suports new line ad carriage return combination', query: '{a}\r\neq\r\n"0"' }, - { name: 'supports tab', query: '{a}\teq\t"0"' }, + processCases((query: string) => new QuerySyntaxTree(query), [ + { name: 'suports new line', query: '{a}\neq\n"0"', target: data0, valid: true, evaluate: true }, + { name: 'suports carriage return', query: '{a}\req\r"0"', target: data0, valid: true, evaluate: true }, + { name: 'suports new line ad carriage return combination', target: data0, query: '{a}\r\neq\r\n"0"', valid: true, evaluate: true }, + { name: 'supports tab', query: '{a}\teq\t"0"', target: data0, valid: true, evaluate: true }, // some random non-standard whitespace character from https://en.wikipedia.org/wiki/Whitespace_character - { name: 'supports ogham space mark', query: '{a}\u1680eq\u1680"0"' }, - { name: 'supports all', query: '{a}\r\n\t\u1680eq\r\n\t\u1680"0"' } - ]; - - R.forEach(c => { - it(c.name, () => { - const tree = new QuerySyntaxTree(c.query); - - expect(tree.isValid).to.equal(true); - expect(tree.evaluate(data0)).to.equal(true); - }); - }, cases); + { name: 'supports ogham space mark', query: '{a}\u1680eq\u1680"0"', target: data0, valid: true, evaluate: true }, + { name: 'supports all', query: '{a}\r\n\t\u1680eq\r\n\t\u1680"0"', target: data0, valid: true, evaluate: true } + ]); }); describe('operands', () => { From 67dcdb3ef73640806afd5a82a7b37c47246b1e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 15:31:44 -0400 Subject: [PATCH 16/32] - reduce cypress memory usage (ts mappings) --- webpack.test.config.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/webpack.test.config.js b/webpack.test.config.js index b1f954194..1928de87b 100644 --- a/webpack.test.config.js +++ b/webpack.test.config.js @@ -1,6 +1,14 @@ -module.exports = require('./.config/webpack/base.js')( +const config = require('./.config/webpack/base.js')( { definitions: ['TEST_COPY_PASTE'] }, 'development' -); \ No newline at end of file +); + +config.module.rules.forEach(rule => { + if (rule.loader) { + rule.loader = rule.loader.replace('ts-loader', `ts-loader?${JSON.stringify({ transpileOnly: true })}`); + } +}); + +module.exports = config; \ No newline at end of file From 598071455ca4e695db15b3a1d1274cf65942dc54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 15:53:16 -0400 Subject: [PATCH 17/32] - revert unused changes --- src/core/syntax-tree/index.ts | 6 ------ src/core/syntax-tree/lexicon.ts | 4 ---- 2 files changed, 10 deletions(-) diff --git a/src/core/syntax-tree/index.ts b/src/core/syntax-tree/index.ts index 442217333..4ad325a39 100644 --- a/src/core/syntax-tree/index.ts +++ b/src/core/syntax-tree/index.ts @@ -57,12 +57,6 @@ export default class SyntaxTree { public readonly config: TConfig, postProcessor: (res: ILexerResult) => ILexerResult = res => res ) { - this.lexicon = R.map(lexeme => R.merge(lexeme, { - evaluate: lexeme.boundedEvaluate ? - lexeme.boundedEvaluate.bind(undefined, config) : - lexeme.evaluate - }), lexicon); - this.lexerResult = postProcessor(configurableLexer(this.lexicon, this.query, this.config)); this.syntaxerResult = syntaxer(this.lexerResult); } diff --git a/src/core/syntax-tree/lexicon.ts b/src/core/syntax-tree/lexicon.ts index 6896f5884..861a6d31e 100644 --- a/src/core/syntax-tree/lexicon.ts +++ b/src/core/syntax-tree/lexicon.ts @@ -24,9 +24,6 @@ export interface IUnboundedLexeme { syntaxer?: (lexs: any[], pivot: any, pivotIndex: number) => any; } -type Evaluate = - ((config: TConfig, target: any, tree: ISyntaxTree) => boolean); - type If = (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean) | ((lexemes: ILexemeResult[], previous: ILexemeResult, config: TConfig) => boolean); @@ -36,7 +33,6 @@ type Terminal = boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult, config: TConfig) => boolean); export interface ILexeme extends IUnboundedLexeme { - boundedEvaluate?: Evaluate; if: If; terminal: Terminal; } From 91a16cba3da631b4b404c0df52a1d4ffbba9229e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 16:09:32 -0400 Subject: [PATCH 18/32] revert generic lexicon changes --- src/core/syntax-tree/index.ts | 13 +++++---- src/core/syntax-tree/lexer.ts | 26 +++++++----------- src/core/syntax-tree/lexicon.ts | 17 +++++------- src/core/syntax-tree/syntaxer.ts | 6 ++--- .../syntax-tree/MultiColumnsSyntaxTree.ts | 2 +- src/dash-table/syntax-tree/QuerySyntaxTree.ts | 2 +- .../syntax-tree/SingleColumnSyntaxTree.ts | 27 ++++++++++++------- src/dash-table/syntax-tree/lexicon/column.ts | 10 +++---- 8 files changed, 48 insertions(+), 55 deletions(-) diff --git a/src/core/syntax-tree/index.ts b/src/core/syntax-tree/index.ts index 4ad325a39..bf3e3568f 100644 --- a/src/core/syntax-tree/index.ts +++ b/src/core/syntax-tree/index.ts @@ -1,7 +1,7 @@ import * as R from 'ramda'; import Logger from 'core/Logger'; -import { ILexerResult, configurableLexer } from 'core/syntax-tree/lexer'; +import lexer, { ILexerResult } from 'core/syntax-tree/lexer'; import syntaxer, { ISyntaxerResult, ISyntaxTree } from 'core/syntax-tree/syntaxer'; import { Lexicon } from './lexicon'; @@ -39,8 +39,8 @@ function toStructure(tree: ISyntaxTree): IStructure { return res; } -export default class SyntaxTree { - protected lexerResult: ILexerResult; +export default class SyntaxTree { + protected lexerResult: ILexerResult; protected syntaxerResult: ISyntaxerResult; get isValid() { @@ -52,12 +52,11 @@ export default class SyntaxTree { } constructor( - public readonly lexicon: Lexicon, + public readonly lexicon: Lexicon, public readonly query: string, - public readonly config: TConfig, - postProcessor: (res: ILexerResult) => ILexerResult = res => res + postProcessor: (res: ILexerResult) => ILexerResult = res => res ) { - this.lexerResult = postProcessor(configurableLexer(this.lexicon, this.query, this.config)); + this.lexerResult = postProcessor(lexer(this.lexicon, this.query)); this.syntaxerResult = syntaxer(this.lexerResult); } diff --git a/src/core/syntax-tree/lexer.ts b/src/core/syntax-tree/lexer.ts index 95c713a52..c0fa303f5 100644 --- a/src/core/syntax-tree/lexer.ts +++ b/src/core/syntax-tree/lexer.ts @@ -2,27 +2,19 @@ import * as R from 'ramda'; import { ILexeme, Lexicon } from 'core/syntax-tree/lexicon'; -export interface ILexerResult { - lexemes: ILexemeResult[]; +export interface ILexerResult { + lexemes: ILexemeResult[]; valid: boolean; error?: string; } -export interface ILexemeResult { - lexeme: ILexeme; +export interface ILexemeResult { + lexeme: ILexeme; value?: string; } -export default function lexer(lexicon: Lexicon, query: string) { - return configurableLexer(lexicon, query, undefined); -} - -export function configurableLexer( - lexicon: Lexicon, - query: string, - config: TConfig -): ILexerResult { - let result: ILexemeResult[] = []; +export default function lexer(lexicon: Lexicon, query: string): ILexerResult { + let result: ILexemeResult[] = []; while (query.length) { query = query.replace(/^\s+/, ''); @@ -30,10 +22,10 @@ export function configurableLexer( const previous = result.slice(-1)[0]; const previousLexeme = previous ? previous.lexeme : null; - let lexemes: ILexeme[] = lexicon.filter(lexeme => + let lexemes: ILexeme[] = lexicon.filter(lexeme => lexeme.if && (!Array.isArray(lexeme.if) ? - lexeme.if(result, previous, config) : + lexeme.if(result, previous) : (previousLexeme ? lexeme.if && lexeme.if.indexOf(previousLexeme.type) !== -1 : lexeme.if && lexeme.if.indexOf(undefined) !== -1)) @@ -53,7 +45,7 @@ export function configurableLexer( const last = result.slice(-1)[0]; const terminal: boolean = last && (typeof last.lexeme.terminal === 'function' ? - last.lexeme.terminal(result, last, config) : + last.lexeme.terminal(result, last) : last.lexeme.terminal); return { diff --git a/src/core/syntax-tree/lexicon.ts b/src/core/syntax-tree/lexicon.ts index 861a6d31e..a638a0fd2 100644 --- a/src/core/syntax-tree/lexicon.ts +++ b/src/core/syntax-tree/lexicon.ts @@ -24,21 +24,16 @@ export interface IUnboundedLexeme { syntaxer?: (lexs: any[], pivot: any, pivotIndex: number) => any; } -type If = (string | undefined)[] | - ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean) | - ((lexemes: ILexemeResult[], previous: ILexemeResult, config: TConfig) => boolean); +type If = (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); +type Terminal = boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); -type Terminal = boolean | - ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean) | - ((lexemes: ILexemeResult[], previous: ILexemeResult, config: TConfig) => boolean); - -export interface ILexeme extends IUnboundedLexeme { - if: If; - terminal: Terminal; +export interface ILexeme extends IUnboundedLexeme { + if: If; + terminal: Terminal; } export function boundLexeme(lexeme: IUnboundedLexeme) { return { ...lexeme, if: () => false, terminal: false }; } -export type Lexicon = ILexeme[]; \ No newline at end of file +export type Lexicon = ILexeme[]; \ No newline at end of file diff --git a/src/core/syntax-tree/syntaxer.ts b/src/core/syntax-tree/syntaxer.ts index b5507808f..d420747e0 100644 --- a/src/core/syntax-tree/syntaxer.ts +++ b/src/core/syntax-tree/syntaxer.ts @@ -9,14 +9,14 @@ export interface ISyntaxerResult { } export interface ISyntaxTree { - lexeme: ILexeme; + lexeme: ILexeme; block?: ISyntaxTree; left?: ISyntaxTree; right: ISyntaxTree; value: string; } -const parser = (lexs: ILexemeResult[]): ISyntaxTree => { +const parser = (lexs: ILexemeResult[]): ISyntaxTree => { let nesting = 0; const nestedLexs = lexs.map(lex => { @@ -57,7 +57,7 @@ const parser = (lexs: ILexemeResult[]): ISyntaxTree => { } }; -export default (lexerResult: ILexerResult): ISyntaxerResult => { +export default (lexerResult: ILexerResult): ISyntaxerResult => { const { lexemes } = lexerResult; if (!lexerResult.valid) { diff --git a/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts b/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts index 6871867bb..b83d4dd8f 100644 --- a/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts +++ b/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts @@ -9,7 +9,7 @@ import { ILexemeResult } from 'core/syntax-tree/lexer'; export default class MultiColumnsSyntaxTree extends SyntaxTree { constructor(query: string) { - super(columnMultiLexicon, query, undefined); + super(columnMultiLexicon, query); } get isValid() { return super.isValid && diff --git a/src/dash-table/syntax-tree/QuerySyntaxTree.ts b/src/dash-table/syntax-tree/QuerySyntaxTree.ts index 58286a5a5..a309b9d2d 100644 --- a/src/dash-table/syntax-tree/QuerySyntaxTree.ts +++ b/src/dash-table/syntax-tree/QuerySyntaxTree.ts @@ -4,6 +4,6 @@ import queryLexicon from './lexicon/query'; export default class QuerySyntaxTree extends SyntaxTree { constructor(query: string) { - super(queryLexicon, query, undefined); + super(queryLexicon, query); } } \ No newline at end of file diff --git a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts index 4312b541b..83ae98b74 100644 --- a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts +++ b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts @@ -9,21 +9,33 @@ import { equal, RelationalOperator } from './lexeme/relational'; import columnLexicon, { ISingleColumnConfig } from './lexicon/column'; -function isBinary(lexemes: ILexemeResult[]) { +function getDefaultRelationalOperator(type: ColumnType = ColumnType.Any): RelationalOperator { + switch (type) { + case ColumnType.Any: + case ColumnType.Text: + return RelationalOperator.Contains; + case ColumnType.Datetime: + return RelationalOperator.LikeDate; + case ColumnType.Numeric: + return RelationalOperator.Equal; + } +} + +function isBinary(lexemes: ILexemeResult[]) { return lexemes.length === 2; } -function isExpression(lexemes: ILexemeResult[]) { +function isExpression(lexemes: ILexemeResult[]) { return lexemes.length === 1 && lexemes[0].lexeme.type === LexemeType.Expression; } -function isUnary(lexemes: ILexemeResult[]) { +function isUnary(lexemes: ILexemeResult[]) { return lexemes.length === 1 && lexemes[0].lexeme.type === LexemeType.UnaryOperator; } -export function modifyLex(config: ISingleColumnConfig, res: ILexerResult) { +export function modifyLex(config: ISingleColumnConfig, res: ILexerResult) { if (!res.valid) { return res; } @@ -38,9 +50,7 @@ export function modifyLex(config: ISingleColumnConfig, res: ILexerResult { +export default class SingleColumnSyntaxTree extends SyntaxTree { constructor(query: string, config: ISingleColumnConfig) { super( columnLexicon, query, - config, modifyLex.bind(undefined, config) ); } diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index 82e78777a..fd4ca3bbf 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -36,9 +36,7 @@ export interface ISingleColumnConfig { }; } -type LexemeResult = ILexemeResult; - -const lexicon: ILexeme[] = [ +const lexicon: ILexeme[] = [ ...[contains, equal, greaterOrEqual, @@ -50,7 +48,7 @@ const lexicon: ILexeme[] = [ ].map(op => ({ ...op, terminal: false, - if: (_lexs: LexemeResult[], previous: LexemeResult) => !previous + if: (_lexs: ILexemeResult[], previous: ILexemeResult) => !previous })), ...[isBool, isEven, @@ -62,7 +60,7 @@ const lexicon: ILexeme[] = [ isStr ].map(op => ({ ...op, - if: (_lexs: LexemeResult[], previous: LexemeResult) => !previous, + if: (_lexs: ILexemeResult[], previous: ILexemeResult) => !previous, terminal: true })), ...[ @@ -71,7 +69,7 @@ const lexicon: ILexeme[] = [ valueExpression ].map(exp => ({ ...exp, - if: (_lexs: LexemeResult[], previous: LexemeResult) => + if: (_lexs: ILexemeResult[], previous: ILexemeResult) => !previous || R.contains( previous.lexeme.type, [LexemeType.RelationalOperator] From f95cda948129931615ac1ca4191b16059404f441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 16:18:28 -0400 Subject: [PATCH 19/32] single column `any` test update --- tests/cypress/tests/unit/single_column_syntactic_tree_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts index 6878a8e72..aa320d1d5 100644 --- a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts +++ b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts @@ -83,7 +83,7 @@ describe('Single Column Syntax Tree', () => { expect(tree.evaluate({ a: 1 })).to.equal(true); expect(tree.evaluate({ a: 2 })).to.equal(false); - expect(tree.toQueryString()).to.equal('{a} = 1'); + expect(tree.toQueryString()).to.equal('{a} contains 1'); }); it('can be expression with numeric column type', () => { From 4e931869852f07ff3d9095fd8afde9b1e24dac3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 20:08:06 -0400 Subject: [PATCH 20/32] clean up --- src/core/syntax-tree/lexicon.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/core/syntax-tree/lexicon.ts b/src/core/syntax-tree/lexicon.ts index a638a0fd2..6a26e1b77 100644 --- a/src/core/syntax-tree/lexicon.ts +++ b/src/core/syntax-tree/lexicon.ts @@ -24,12 +24,9 @@ export interface IUnboundedLexeme { syntaxer?: (lexs: any[], pivot: any, pivotIndex: number) => any; } -type If = (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); -type Terminal = boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); - export interface ILexeme extends IUnboundedLexeme { - if: If; - terminal: Terminal; + terminal: boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); + if: (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); } export function boundLexeme(lexeme: IUnboundedLexeme) { From 0d730cc3557245c73bfaa6eb842764f03c556179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 20:33:03 -0400 Subject: [PATCH 21/32] simple column query config rework --- src/core/type/index.ts | 2 ++ src/dash-table/components/FilterFactory.tsx | 2 +- .../syntax-tree/SingleColumnSyntaxTree.ts | 17 ++++++++++------- src/dash-table/syntax-tree/index.ts | 10 +++------- src/dash-table/syntax-tree/lexicon/column.ts | 7 ------- 5 files changed, 16 insertions(+), 22 deletions(-) create mode 100644 src/core/type/index.ts diff --git a/src/core/type/index.ts b/src/core/type/index.ts new file mode 100644 index 000000000..f509ef023 --- /dev/null +++ b/src/core/type/index.ts @@ -0,0 +1,2 @@ +export type RequiredPluck = { [r in R]: T[r] }; +export type OptionalPluck = { [r in R]?: T[r] }; diff --git a/src/dash-table/components/FilterFactory.tsx b/src/dash-table/components/FilterFactory.tsx index 6e7ec9813..6ebbdf80b 100644 --- a/src/dash-table/components/FilterFactory.tsx +++ b/src/dash-table/components/FilterFactory.tsx @@ -55,7 +55,7 @@ export default class FilterFactory { const safeColumnId = column.id.toString(); if (value && value.length) { - this.ops.set(safeColumnId, new SingleColumnSyntaxTree(value, { column })); + this.ops.set(safeColumnId, new SingleColumnSyntaxTree(value, column)); } else { this.ops.delete(safeColumnId); } diff --git a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts index 83ae98b74..147d23c81 100644 --- a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts +++ b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts @@ -1,13 +1,14 @@ +import { RequiredPluck, OptionalPluck } from 'core/type'; import SyntaxTree from 'core/syntax-tree'; import { ILexemeResult, ILexerResult } from 'core/syntax-tree/lexer'; import { LexemeType, boundLexeme } from 'core/syntax-tree/lexicon'; -import { ColumnType } from 'dash-table/components/Table/props'; +import { ColumnType, IVisibleColumn } from 'dash-table/components/Table/props'; import operand from './lexeme/operand'; import { equal, RelationalOperator } from './lexeme/relational'; -import columnLexicon, { ISingleColumnConfig } from './lexicon/column'; +import columnLexicon from './lexicon/column'; function getDefaultRelationalOperator(type: ColumnType = ColumnType.Any): RelationalOperator { switch (type) { @@ -35,22 +36,22 @@ function isUnary(lexemes: ILexemeResult[]) { lexemes[0].lexeme.type === LexemeType.UnaryOperator; } -export function modifyLex(config: ISingleColumnConfig, res: ILexerResult) { +export function modifyLex(config: SingleColumnConfig, res: ILexerResult) { if (!res.valid) { return res; } if (isBinary(res.lexemes) || isUnary(res.lexemes)) { res.lexemes = [ - { lexeme: boundLexeme(operand), value: `{${config.column.id}}` }, + { lexeme: boundLexeme(operand), value: `{${config.id}}` }, ...res.lexemes ]; } else if (isExpression(res.lexemes)) { res.lexemes = [ - { lexeme: boundLexeme(operand), value: `{${config.column.id}}` }, + { lexeme: boundLexeme(operand), value: `{${config.id}}` }, { lexeme: boundLexeme(equal), - value: getDefaultRelationalOperator(config.column.type) + value: getDefaultRelationalOperator(config.type) }, ...res.lexemes ]; @@ -59,8 +60,10 @@ export function modifyLex(config: ISingleColumnConfig, res: ILexerResult) { return res; } +type SingleColumnConfig = RequiredPluck & OptionalPluck; + export default class SingleColumnSyntaxTree extends SyntaxTree { - constructor(query: string, config: ISingleColumnConfig) { + constructor(query: string, config: SingleColumnConfig) { super( columnLexicon, query, diff --git a/src/dash-table/syntax-tree/index.ts b/src/dash-table/syntax-tree/index.ts index a3654805f..c650d9237 100644 --- a/src/dash-table/syntax-tree/index.ts +++ b/src/dash-table/syntax-tree/index.ts @@ -39,9 +39,7 @@ export const getSingleColumnMap = ( throw new Error(`column ${sanitizedColumnId} not found`); } - const config = { column }; - - map.set(sanitizedColumnId, new SingleColumnSyntaxTree(s.value, config)); + map.set(sanitizedColumnId, new SingleColumnSyntaxTree(s.value, column)); } else if (s.lexeme.type === LexemeType.RelationalOperator && s.left && s.right) { const sanitizedColumnId = s.left.lexeme.present ? s.left.lexeme.present(s.left) : s.left.value; @@ -50,12 +48,10 @@ export const getSingleColumnMap = ( throw new Error(`column ${sanitizedColumnId} not found`); } - const config = { column }; - if (s.lexeme.present && s.lexeme.present(s) === RelationalOperator.Equal) { - map.set(sanitizedColumnId, new SingleColumnSyntaxTree(`${s.right.value}`, config)); + map.set(sanitizedColumnId, new SingleColumnSyntaxTree(`${s.right.value}`, column)); } else { - map.set(sanitizedColumnId, new SingleColumnSyntaxTree(`${s.value} ${s.right.value}`, config)); + map.set(sanitizedColumnId, new SingleColumnSyntaxTree(`${s.value} ${s.right.value}`, column)); } } }, statements); diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index fd4ca3bbf..bc85e805d 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -28,13 +28,6 @@ import { import { LexemeType, ILexeme } from 'core/syntax-tree/lexicon'; import { ILexemeResult } from 'core/syntax-tree/lexer'; -import { ColumnId, IVisibleColumn } from 'dash-table/components/Table/props'; - -export interface ISingleColumnConfig { - column: Partial & { - id: ColumnId - }; -} const lexicon: ILexeme[] = [ ...[contains, From 4e3fe5f807cae81105dbe49bceb0aacd5abd2355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Wed, 24 Apr 2019 20:43:19 -0400 Subject: [PATCH 22/32] single column query config rework --- .../syntax-tree/SingleColumnSyntaxTree.ts | 2 +- .../tests/unit/dash_table_queries_test.ts | 4 +-- .../unit/single_column_syntactic_tree_test.ts | 34 +++++++------------ 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts index 147d23c81..66854d605 100644 --- a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts +++ b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts @@ -60,7 +60,7 @@ export function modifyLex(config: SingleColumnConfig, res: ILexerResult) { return res; } -type SingleColumnConfig = RequiredPluck & OptionalPluck; +export type SingleColumnConfig = RequiredPluck & OptionalPluck; export default class SingleColumnSyntaxTree extends SyntaxTree { constructor(query: string, config: SingleColumnConfig) { diff --git a/tests/cypress/tests/unit/dash_table_queries_test.ts b/tests/cypress/tests/unit/dash_table_queries_test.ts index ae6d1b447..841565c50 100644 --- a/tests/cypress/tests/unit/dash_table_queries_test.ts +++ b/tests/cypress/tests/unit/dash_table_queries_test.ts @@ -27,9 +27,7 @@ export function processCases(getSyntaxer: (query: string) => SyntaxTree, cases: const getQuerySyntaxTree = (query: string): any => new QuerySyntaxTree(query); const getMultiColumnSyntaxTree = (query: string): any => new MultiColumnsSyntaxTree(query); const getSingleColumnSyntaxTree = (query: string): any => new SingleColumnSyntaxTree(query, { - column: { - id: 'a' - } + id: 'a' }); describe('Dash Table Queries', () => { diff --git a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts index aa320d1d5..d98108f00 100644 --- a/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts +++ b/tests/cypress/tests/unit/single_column_syntactic_tree_test.ts @@ -1,33 +1,25 @@ import { SingleColumnSyntaxTree } from 'dash-table/syntax-tree'; import { ColumnType } from 'dash-table/components/Table/props'; -import { ISingleColumnConfig } from 'dash-table/syntax-tree/lexicon/column'; +import { SingleColumnConfig } from 'dash-table/syntax-tree/SingleColumnSyntaxTree'; -const COLUMN_ANY: ISingleColumnConfig = { - column: { - id: 'a', - type: ColumnType.Any - } +const COLUMN_ANY: SingleColumnConfig = { + id: 'a', + type: ColumnType.Any }; -const COLUMN_NUMERIC: ISingleColumnConfig = { - column: { - id: 'a', - type: ColumnType.Numeric - } +const COLUMN_NUMERIC: SingleColumnConfig = { + id: 'a', + type: ColumnType.Numeric }; -const COLUMN_TEXT: ISingleColumnConfig = { - column: { - id: 'a', - type: ColumnType.Text - } +const COLUMN_TEXT: SingleColumnConfig = { + id: 'a', + type: ColumnType.Text }; -const COLUMN_UNDEFINED: ISingleColumnConfig = { - column: { - id: 'a', - type: undefined - } +const COLUMN_UNDEFINED: SingleColumnConfig = { + id: 'a', + type: undefined }; describe('Single Column Syntax Tree', () => { From 297706ba9ef3bab77ce69ce2019b701cde1c7b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 08:43:25 -0400 Subject: [PATCH 23/32] rename `in` -> `like` --- .../syntax-tree/lexeme/relational.ts | 4 +- .../tests/unit/dash_table_queries_test.ts | 48 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/dash-table/syntax-tree/lexeme/relational.ts b/src/dash-table/syntax-tree/lexeme/relational.ts index dc344ad50..ad975b906 100644 --- a/src/dash-table/syntax-tree/lexeme/relational.ts +++ b/src/dash-table/syntax-tree/lexeme/relational.ts @@ -39,7 +39,7 @@ export enum RelationalOperator { GreaterThan = '>', LessOrEqual = '<=', LessThan = '<', - LikeDate = 'in', + LikeDate = 'like', NotEqual = '!=' } @@ -98,7 +98,7 @@ export const likeDate: IUnboundedLexeme = R.merge({ normalizedOp.indexOf(normalizedExp) === 0; }), subType: RelationalOperator.LikeDate, - regexp: /^(in)/i + regexp: /^(like)/i }, LEXEME_BASE); export const lessOrEqual: IUnboundedLexeme = R.merge({ diff --git a/tests/cypress/tests/unit/dash_table_queries_test.ts b/tests/cypress/tests/unit/dash_table_queries_test.ts index 841565c50..60742db52 100644 --- a/tests/cypress/tests/unit/dash_table_queries_test.ts +++ b/tests/cypress/tests/unit/dash_table_queries_test.ts @@ -70,30 +70,30 @@ describe('Dash Table Queries', () => { describe('in', () => { processCases(c.syntaxer, [ - { name: '0yyy in "0"', query: `${c.hideOperand ? '' : '{a} '}in "0"`, target: { a: '0987' }, valid: true, evaluate: false }, - { name: '0yyy in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}in "0987"`, target: { a: '0987' }, valid: true, evaluate: true }, - { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2006"`, target: { a: '2005' }, valid: true, evaluate: false }, - { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005' }, valid: true, evaluate: true }, - { name: 'yyyy in yyyy', query: `${c.hideOperand ? '' : '{a} '}in 2005`, target: { a: '2005' }, valid: true, evaluate: false }, - { name: 'yyyy-mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}in "2005"`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, - { name: 'yyyy in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005' }, valid: true, evaluate: false }, - { name: 'yyyy-mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm in "yyyy-mm"', query: `${c.hideOperand ? '' : '{a} '}in "2005-01"`, target: { a: '2005-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}in 2005-01`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx', query: `${c.hideOperand ? '' : '{a} '}in 2005-01-01T10:00:00.000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx000', query: `${c.hideOperand ? '' : '{a} '}in 2005-01-01T10:00:00.000000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx111', query: `${c.hideOperand ? '' : '{a} '}in 2005-01-01T10:00:00.000111`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false } + { name: '0yyy in "0"', query: `${c.hideOperand ? '' : '{a} '}like "0"`, target: { a: '0987' }, valid: true, evaluate: false }, + { name: '0yyy in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}like "0987"`, target: { a: '0987' }, valid: true, evaluate: true }, + { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2006"`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005' }, valid: true, evaluate: true }, + { name: 'yyyy in yyyy', query: `${c.hideOperand ? '' : '{a} '}like 2005`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy-mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, + { name: 'yyyy in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy-mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm in "yyyy-mm"', query: `${c.hideOperand ? '' : '{a} '}like "2005-01"`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx000', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx111', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000111`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false } ]); }); }); From 606a7c6c3eaac003d73fe1c676123da4493770a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 11:09:17 -0400 Subject: [PATCH 24/32] - remove `operand` in favor of `fieldExpression` --- .../syntax-tree/SingleColumnSyntaxTree.ts | 6 +- .../syntax-tree/lexeme/expression.ts | 28 +++-- src/dash-table/syntax-tree/lexeme/operand.ts | 26 ----- src/dash-table/syntax-tree/lexicon/column.ts | 20 ++-- .../syntax-tree/lexicon/columnMulti.ts | 50 +++------ src/dash-table/syntax-tree/lexicon/index.ts | 69 ++++++++++++ src/dash-table/syntax-tree/lexicon/query.ts | 103 +++++------------- tests/cypress/tests/unit/lexeme_test.ts | 21 ---- 8 files changed, 141 insertions(+), 182 deletions(-) delete mode 100644 src/dash-table/syntax-tree/lexeme/operand.ts create mode 100644 src/dash-table/syntax-tree/lexicon/index.ts diff --git a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts index 66854d605..01707c725 100644 --- a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts +++ b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts @@ -5,7 +5,7 @@ import { LexemeType, boundLexeme } from 'core/syntax-tree/lexicon'; import { ColumnType, IVisibleColumn } from 'dash-table/components/Table/props'; -import operand from './lexeme/operand'; +import { fieldExpression } from './lexeme/expression'; import { equal, RelationalOperator } from './lexeme/relational'; import columnLexicon from './lexicon/column'; @@ -43,12 +43,12 @@ export function modifyLex(config: SingleColumnConfig, res: ILexerResult) { if (isBinary(res.lexemes) || isUnary(res.lexemes)) { res.lexemes = [ - { lexeme: boundLexeme(operand), value: `{${config.id}}` }, + { lexeme: boundLexeme(fieldExpression), value: `{${config.id}}` }, ...res.lexemes ]; } else if (isExpression(res.lexemes)) { res.lexemes = [ - { lexeme: boundLexeme(operand), value: `{${config.id}}` }, + { lexeme: boundLexeme(fieldExpression), value: `{${config.id}}` }, { lexeme: boundLexeme(equal), value: getDefaultRelationalOperator(config.type) diff --git a/src/dash-table/syntax-tree/lexeme/expression.ts b/src/dash-table/syntax-tree/lexeme/expression.ts index 8dad633ee..730586a6a 100644 --- a/src/dash-table/syntax-tree/lexeme/expression.ts +++ b/src/dash-table/syntax-tree/lexeme/expression.ts @@ -1,19 +1,31 @@ import isNumeric from 'fast-isnumeric'; -import * as R from 'ramda'; import { LexemeType, IUnboundedLexeme } from 'core/syntax-tree/lexicon'; import { ISyntaxTree } from 'core/syntax-tree/syntaxer'; -import operand from './operand'; +const FIELD_REGEX = /^{(([^{}\\]|\\.)+)}/; const STRING_REGEX = /^(('([^'\\]|\\.)+')|("([^"\\]|\\.)+")|(`([^`\\]|\\.)+`))/; const VALUE_REGEX = /^(([^\s'"`{}()\\]|\\.)+)(?:[\s)]|$)/; -export const fieldExpression: IUnboundedLexeme = R.merge( - operand, { - subType: 'field', - type: LexemeType.Expression - } -); +const getField = ( + value: string +) => value + .slice(1, value.length - 1) + .replace(/\\(.)/g, '$1'); + +export const fieldExpression: IUnboundedLexeme = { + present: (tree: ISyntaxTree) => getField(tree.value), + resolve: (target: any, tree: ISyntaxTree) => { + if (FIELD_REGEX.test(tree.value)) { + return target[getField(tree.value)]; + } else { + throw new Error(); + } + }, + regexp: FIELD_REGEX, + subType: 'field', + type: LexemeType.Expression +}; const getString = ( value: string diff --git a/src/dash-table/syntax-tree/lexeme/operand.ts b/src/dash-table/syntax-tree/lexeme/operand.ts deleted file mode 100644 index 5fe72feea..000000000 --- a/src/dash-table/syntax-tree/lexeme/operand.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { LexemeType, IUnboundedLexeme } from 'core/syntax-tree/lexicon'; -import { ISyntaxTree } from 'core/syntax-tree/syntaxer'; - -const FIELD_REGEX = /^{(([^{}\\]|\\.)+)}/; - -const getField = ( - value: string -) => value - .slice(1, value.length - 1) - .replace(/\\(.)/g, '$1'); - -const operand: IUnboundedLexeme = { - present: (tree: ISyntaxTree) => getField(tree.value), - resolve: (target: any, tree: ISyntaxTree) => { - if (FIELD_REGEX.test(tree.value)) { - return target[getField(tree.value)]; - } else { - throw new Error(); - } - }, - regexp: FIELD_REGEX, - subType: 'field', - type: LexemeType.Operand -}; - -export default operand; \ No newline at end of file diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index bc85e805d..19d4a5daa 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -1,5 +1,3 @@ -import * as R from 'ramda'; - import { fieldExpression, stringExpression, @@ -26,8 +24,12 @@ import { isStr } from '../lexeme/unary'; -import { LexemeType, ILexeme } from 'core/syntax-tree/lexicon'; -import { ILexemeResult } from 'core/syntax-tree/lexer'; +import { + ifExpression, + ifLeading +} from '.'; + +import { ILexeme } from 'core/syntax-tree/lexicon'; const lexicon: ILexeme[] = [ ...[contains, @@ -41,7 +43,7 @@ const lexicon: ILexeme[] = [ ].map(op => ({ ...op, terminal: false, - if: (_lexs: ILexemeResult[], previous: ILexemeResult) => !previous + if: ifLeading })), ...[isBool, isEven, @@ -53,7 +55,7 @@ const lexicon: ILexeme[] = [ isStr ].map(op => ({ ...op, - if: (_lexs: ILexemeResult[], previous: ILexemeResult) => !previous, + if: ifLeading, terminal: true })), ...[ @@ -62,11 +64,7 @@ const lexicon: ILexeme[] = [ valueExpression ].map(exp => ({ ...exp, - if: (_lexs: ILexemeResult[], previous: ILexemeResult) => - !previous || R.contains( - previous.lexeme.type, - [LexemeType.RelationalOperator] - ), + if: ifExpression, terminal: true })) ]; diff --git a/src/dash-table/syntax-tree/lexicon/columnMulti.ts b/src/dash-table/syntax-tree/lexicon/columnMulti.ts index b379924c6..74fd87206 100644 --- a/src/dash-table/syntax-tree/lexicon/columnMulti.ts +++ b/src/dash-table/syntax-tree/lexicon/columnMulti.ts @@ -1,4 +1,4 @@ -import * as R from 'ramda'; +import { ILexeme } from 'core/syntax-tree/lexicon'; import { fieldExpression, @@ -8,7 +8,6 @@ import { import { and } from '../lexeme/logical'; -import operand from '../lexeme/operand'; import { contains, equal, @@ -30,29 +29,18 @@ import { isStr } from '../lexeme/unary'; -import { ILexeme, LexemeType } from 'core/syntax-tree/lexicon'; -import { ILexemeResult } from 'core/syntax-tree/lexer'; +import { + ifExpression, + ifLogicalOperator, + ifRelationalOperator, + ifUnaryOperator, + isTerminalExpression +} from '.'; const lexicon: ILexeme[] = [ { ...and, - if: (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( - previous.lexeme.type, - [ - LexemeType.Expression, - LexemeType.UnaryOperator - ] - ), - terminal: false - }, - { - ...operand, - if: (_: ILexemeResult[], previous: ILexemeResult) => - !previous || R.contains( - previous.lexeme.type, - [LexemeType.LogicalOperator] - ), + if: ifLogicalOperator, terminal: false }, ...[contains, @@ -65,11 +53,7 @@ const lexicon: ILexeme[] = [ notEqual ].map(op => ({ ...op, - if: (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( - previous.lexeme.type, - [LexemeType.Operand] - ), + if: ifRelationalOperator, terminal: false })), ...[isBool, @@ -82,11 +66,7 @@ const lexicon: ILexeme[] = [ isStr ].map(op => ({ ...op, - if: (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( - previous.lexeme.type, - [LexemeType.Operand] - ), + if: ifUnaryOperator, terminal: true })), ...[ @@ -95,12 +75,8 @@ const lexicon: ILexeme[] = [ valueExpression ].map(exp => ({ ...exp, - if: (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( - previous.lexeme.type, - [LexemeType.RelationalOperator] - ), - terminal: true + if: ifExpression, + terminal: isTerminalExpression })) ]; diff --git a/src/dash-table/syntax-tree/lexicon/index.ts b/src/dash-table/syntax-tree/lexicon/index.ts new file mode 100644 index 000000000..9a4932590 --- /dev/null +++ b/src/dash-table/syntax-tree/lexicon/index.ts @@ -0,0 +1,69 @@ +import * as R from 'ramda'; + +import { ILexemeResult } from 'core/syntax-tree/lexer'; +import { LexemeType } from 'core/syntax-tree/lexicon'; + +const nestingReducer = R.reduce( + (nesting, l) => nesting + (l.lexeme.nesting || 0) +); + +export const isTerminal = (lexemes: ILexemeResult[], previous: ILexemeResult) => + previous && nestingReducer(0, lexemes) === 0; + +export const isTerminalExpression = (lexemes: ILexemeResult[], previous: ILexemeResult) => + isTerminal(lexemes, previous) && + R.contains(previous.lexeme.type, [ + LexemeType.RelationalOperator + ]); + +export const ifBlockClose = (lexemes: ILexemeResult[], previous: ILexemeResult) => + previous && R.contains( + previous.lexeme.type, + [ + LexemeType.BlockClose, + LexemeType.BlockOpen, + LexemeType.Expression, + LexemeType.UnaryOperator + ] + ) && nestingReducer(0, lexemes) > 0; + +export const ifBlockOpen = (_: ILexemeResult[], previous: ILexemeResult) => + !previous || R.contains( + previous.lexeme.type, + [ + LexemeType.BlockOpen, + LexemeType.LogicalOperator, + LexemeType.UnaryOperator + ] + ); + +export const ifExpression = (_: ILexemeResult[], previous: ILexemeResult) => + previous && R.contains( + previous.lexeme.type, + [ + LexemeType.BlockOpen, + LexemeType.LogicalOperator, + LexemeType.RelationalOperator + ] + ); + +export const ifLeading = (_lexs: ILexemeResult[], previous: ILexemeResult) => + !previous; + +export const ifLogicalOperator = (_: ILexemeResult[], previous: ILexemeResult) => + previous && R.contains( + previous.lexeme.type, + [ + LexemeType.BlockClose, + LexemeType.Expression, + LexemeType.UnaryOperator + ] + ); + +export const ifRelationalOperator = (_: ILexemeResult[], previous: ILexemeResult) => + previous && R.contains( + previous.lexeme.type, + [LexemeType.Operand] + ); + +export const ifUnaryOperator = ifRelationalOperator; \ No newline at end of file diff --git a/src/dash-table/syntax-tree/lexicon/query.ts b/src/dash-table/syntax-tree/lexicon/query.ts index 4ddd9ddb2..af5d5e135 100644 --- a/src/dash-table/syntax-tree/lexicon/query.ts +++ b/src/dash-table/syntax-tree/lexicon/query.ts @@ -1,5 +1,8 @@ import * as R from 'ramda'; +import { ILexemeResult } from 'core/syntax-tree/lexer'; +import { LexemeType, ILexeme } from 'core/syntax-tree/lexicon'; + import { blockClose, blockOpen @@ -13,7 +16,6 @@ import { and, or } from '../lexeme/logical'; -import operand from '../lexeme/operand'; import { contains, equal, @@ -36,86 +38,42 @@ import { not } from '../lexeme/unary'; -import { ILexemeResult } from 'core/syntax-tree/lexer'; -import { LexemeType, ILexeme } from 'core/syntax-tree/lexicon'; - -const nestingReducer = R.reduce( - (nesting, l) => nesting + (l.lexeme.nesting || 0) -); - -const isTerminal = (lexemes: ILexemeResult[], previous: ILexemeResult) => - previous && nestingReducer(0, lexemes) === 0; - -const ifExpression = (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( - previous.lexeme.type, - [LexemeType.RelationalOperator] - ); +import { + ifBlockClose, + ifBlockOpen, + ifExpression, + ifLogicalOperator, + ifRelationalOperator, + ifUnaryOperator, + isTerminal, + isTerminalExpression +} from '.'; -const ifLogicalOperator = (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( +const ifNotUnaryOperator = (_: ILexemeResult[], previous: ILexemeResult) => + !previous || R.contains( previous.lexeme.type, [ - LexemeType.BlockClose, - LexemeType.Expression, + LexemeType.LogicalOperator, LexemeType.UnaryOperator ] ); -const ifOperator = (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( - previous.lexeme.type, - [LexemeType.Operand] - ); - const lexicon: ILexeme[] = [ - { - ...and, - if: ifLogicalOperator, - terminal: false - }, - { - ...or, + ...[and, + or + ].map(op => ({ + ...op, if: ifLogicalOperator, terminal: false - }, + })), { ...blockClose, - if: (lexemes: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( - previous.lexeme.type, - [ - LexemeType.BlockClose, - LexemeType.BlockOpen, - LexemeType.Expression, - LexemeType.UnaryOperator - ] - ) && nestingReducer(0, lexemes) > 0, + if: ifBlockClose, terminal: isTerminal }, { ...blockOpen, - if: (_: ILexemeResult[], previous: ILexemeResult) => - !previous || R.contains( - previous.lexeme.type, - [ - LexemeType.BlockOpen, - LexemeType.LogicalOperator, - LexemeType.UnaryOperator - ] - ), - terminal: false - }, - { - ...operand, - if: (_: ILexemeResult[], previous: ILexemeResult) => - !previous || R.contains( - previous.lexeme.type, - [ - LexemeType.BlockOpen, - LexemeType.LogicalOperator - ] - ), + if: ifBlockOpen, terminal: false }, ...[contains, @@ -128,7 +86,7 @@ const lexicon: ILexeme[] = [ notEqual ].map(op => ({ ...op, - if: ifOperator, + if: ifRelationalOperator, terminal: false })), ...[isBool, @@ -141,19 +99,12 @@ const lexicon: ILexeme[] = [ isStr ].map(op => ({ ...op, - if: ifOperator, + if: ifUnaryOperator, terminal: isTerminal })), { ...not, - if: (_: ILexemeResult[], previous: ILexemeResult) => - !previous || R.contains( - previous.lexeme.type, - [ - LexemeType.LogicalOperator, - LexemeType.UnaryOperator - ] - ), + if: ifNotUnaryOperator, terminal: false }, ...[ @@ -163,7 +114,7 @@ const lexicon: ILexeme[] = [ ].map(exp => ({ ...exp, if: ifExpression, - terminal: isTerminal + terminal: isTerminalExpression })) ]; diff --git a/tests/cypress/tests/unit/lexeme_test.ts b/tests/cypress/tests/unit/lexeme_test.ts index 5c878e82c..d924c4e9a 100644 --- a/tests/cypress/tests/unit/lexeme_test.ts +++ b/tests/cypress/tests/unit/lexeme_test.ts @@ -3,7 +3,6 @@ import { stringExpression, valueExpression } from 'dash-table/syntax-tree/lexeme/expression'; -import operand from 'dash-table/syntax-tree/lexeme/operand'; import { ISyntaxTree } from 'core/syntax-tree/syntaxer'; describe('expression', () => { @@ -94,24 +93,4 @@ describe('expression', () => { expect(valueExpression.resolve.bind(undefined, {}, { value: '}' } as ISyntaxTree)).to.throw(Error); } }); -}); - -describe('operand', () => { - it('resolves values', () => { - expect(!!operand.resolve).to.equal(true); - expect(typeof operand.resolve).to.equal('function'); - - if (operand.resolve) { - expect(operand.resolve.bind(undefined, {}, { value: 'abc' } as ISyntaxTree)).to.throw(Error); - expect(operand.resolve.bind(undefined, {}, { value: '123' } as ISyntaxTree)).to.throw(Error); - expect(operand.resolve.bind(undefined, {}, { value: '{abc' } as ISyntaxTree)).to.throw(Error); - expect(operand.resolve.bind(undefined, {}, { value: 'abc}' } as ISyntaxTree)).to.throw(Error); - expect(operand.resolve.bind(undefined, {}, { value: '{{abc}}' } as ISyntaxTree)).to.throw(Error); - - expect(operand.resolve({ abc: 3 }, { value: '{abc}' } as ISyntaxTree)).to.equal(3); - expect(operand.resolve({ ['a bc']: 3 }, { value: '{a bc}' } as ISyntaxTree)).to.equal(3); - expect(operand.resolve({ ['{abc}']: 3 }, { value: '{\\{abc\\}}' } as ISyntaxTree)).to.equal(3); - expect(operand.resolve({ ['"abc"']: 3 }, { value: '{"abc"}' } as ISyntaxTree)).to.equal(3); - } - }); }); \ No newline at end of file From 3f6fd78026ff32df37c3e0d1905538eb8c272be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 12:21:13 -0400 Subject: [PATCH 25/32] - fix expression terminal logic - fix terminal logic bug in lexer - fix tests impacted by operand/expression rework --- src/core/syntax-tree/lexer.ts | 18 +++++++---- src/core/syntax-tree/lexicon.ts | 7 ++--- .../syntax-tree/MultiColumnsSyntaxTree.ts | 2 +- src/dash-table/syntax-tree/lexicon/column.ts | 4 +-- src/dash-table/syntax-tree/lexicon/index.ts | 30 ++++++++++--------- src/dash-table/syntax-tree/lexicon/query.ts | 2 +- .../tests/unit/dash_table_queries_test.ts | 2 +- .../tests/unit/query_syntactic_tree_test.ts | 8 ++--- 8 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/core/syntax-tree/lexer.ts b/src/core/syntax-tree/lexer.ts index c0fa303f5..2912f8258 100644 --- a/src/core/syntax-tree/lexer.ts +++ b/src/core/syntax-tree/lexer.ts @@ -42,14 +42,20 @@ export default function lexer(lexicon: Lexicon, query: string): ILexerResult { query = query.substring(value.length); } - const last = result.slice(-1)[0]; - - const terminal: boolean = last && (typeof last.lexeme.terminal === 'function' ? - last.lexeme.terminal(result, last) : - last.lexeme.terminal); + const [terminalPrevious, last] = [ + undefined, + undefined, + ...result.slice(-2) + ].slice(-2); + + const terminal: boolean = !last || + (typeof last.lexeme.terminal === 'function' ? + last.lexeme.terminal(result, terminalPrevious) : + last.lexeme.terminal + ); return { lexemes: result, - valid: !last || terminal + valid: terminal }; } \ No newline at end of file diff --git a/src/core/syntax-tree/lexicon.ts b/src/core/syntax-tree/lexicon.ts index 6a26e1b77..f1bb0907c 100644 --- a/src/core/syntax-tree/lexicon.ts +++ b/src/core/syntax-tree/lexicon.ts @@ -7,8 +7,7 @@ export enum LexemeType { LogicalOperator = 'logical-operator', RelationalOperator = 'relational-operator', UnaryOperator = 'unary-operator', - Expression = 'expression', - Operand = 'operand' + Expression = 'expression' } export interface IUnboundedLexeme { @@ -25,8 +24,8 @@ export interface IUnboundedLexeme { } export interface ILexeme extends IUnboundedLexeme { - terminal: boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); - if: (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean); + terminal: boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult | undefined) => boolean); + if: (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult | undefined) => boolean); } export function boundLexeme(lexeme: IUnboundedLexeme) { diff --git a/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts b/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts index b83d4dd8f..99ca389a7 100644 --- a/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts +++ b/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts @@ -43,7 +43,7 @@ export default class MultiColumnsSyntaxTree extends SyntaxTree { const fields = R.map( (item: ILexemeResult) => item.value, R.filter( - i => i.lexeme.type === LexemeType.Operand, + i => i.lexeme.type === LexemeType.Expression, this.lexerResult.lexemes ) ); diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index 19d4a5daa..ed01851e8 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -42,8 +42,8 @@ const lexicon: ILexeme[] = [ notEqual ].map(op => ({ ...op, - terminal: false, - if: ifLeading + if: ifLeading, + terminal: false })), ...[isBool, isEven, diff --git a/src/dash-table/syntax-tree/lexicon/index.ts b/src/dash-table/syntax-tree/lexicon/index.ts index 9a4932590..af4e92fa4 100644 --- a/src/dash-table/syntax-tree/lexicon/index.ts +++ b/src/dash-table/syntax-tree/lexicon/index.ts @@ -7,17 +7,18 @@ const nestingReducer = R.reduce( (nesting, l) => nesting + (l.lexeme.nesting || 0) ); -export const isTerminal = (lexemes: ILexemeResult[], previous: ILexemeResult) => - previous && nestingReducer(0, lexemes) === 0; +export const isTerminal = (lexemes: ILexemeResult[], _: ILexemeResult | undefined) => + nestingReducer(0, lexemes) === 0; -export const isTerminalExpression = (lexemes: ILexemeResult[], previous: ILexemeResult) => +export const isTerminalExpression = (lexemes: ILexemeResult[], previous: ILexemeResult | undefined) => isTerminal(lexemes, previous) && + !!previous && R.contains(previous.lexeme.type, [ LexemeType.RelationalOperator ]); -export const ifBlockClose = (lexemes: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( +export const ifBlockClose = (lexemes: ILexemeResult[], previous: ILexemeResult | undefined) => + !!previous && R.contains( previous.lexeme.type, [ LexemeType.BlockClose, @@ -27,7 +28,7 @@ export const ifBlockClose = (lexemes: ILexemeResult[], previous: ILexemeResult) ] ) && nestingReducer(0, lexemes) > 0; -export const ifBlockOpen = (_: ILexemeResult[], previous: ILexemeResult) => +export const ifBlockOpen = (_: ILexemeResult[], previous: ILexemeResult | undefined) => !previous || R.contains( previous.lexeme.type, [ @@ -37,8 +38,8 @@ export const ifBlockOpen = (_: ILexemeResult[], previous: ILexemeResult) => ] ); -export const ifExpression = (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( +export const ifExpression = (_: ILexemeResult[], previous: ILexemeResult | undefined) => { + return !previous || R.contains( previous.lexeme.type, [ LexemeType.BlockOpen, @@ -46,12 +47,13 @@ export const ifExpression = (_: ILexemeResult[], previous: ILexemeResult) => LexemeType.RelationalOperator ] ); +}; -export const ifLeading = (_lexs: ILexemeResult[], previous: ILexemeResult) => +export const ifLeading = (_lexs: ILexemeResult[], previous: ILexemeResult | undefined) => !previous; -export const ifLogicalOperator = (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( +export const ifLogicalOperator = (_: ILexemeResult[], previous: ILexemeResult | undefined) => + !!previous && R.contains( previous.lexeme.type, [ LexemeType.BlockClose, @@ -60,10 +62,10 @@ export const ifLogicalOperator = (_: ILexemeResult[], previous: ILexemeResult) = ] ); -export const ifRelationalOperator = (_: ILexemeResult[], previous: ILexemeResult) => - previous && R.contains( +export const ifRelationalOperator = (_: ILexemeResult[], previous: ILexemeResult | undefined) => + !!previous && R.contains( previous.lexeme.type, - [LexemeType.Operand] + [LexemeType.Expression] ); export const ifUnaryOperator = ifRelationalOperator; \ No newline at end of file diff --git a/src/dash-table/syntax-tree/lexicon/query.ts b/src/dash-table/syntax-tree/lexicon/query.ts index af5d5e135..6049caf0a 100644 --- a/src/dash-table/syntax-tree/lexicon/query.ts +++ b/src/dash-table/syntax-tree/lexicon/query.ts @@ -49,7 +49,7 @@ import { isTerminalExpression } from '.'; -const ifNotUnaryOperator = (_: ILexemeResult[], previous: ILexemeResult) => +const ifNotUnaryOperator = (_: ILexemeResult[], previous: ILexemeResult | undefined) => !previous || R.contains( previous.lexeme.type, [ diff --git a/tests/cypress/tests/unit/dash_table_queries_test.ts b/tests/cypress/tests/unit/dash_table_queries_test.ts index 60742db52..6c4bebc31 100644 --- a/tests/cypress/tests/unit/dash_table_queries_test.ts +++ b/tests/cypress/tests/unit/dash_table_queries_test.ts @@ -68,7 +68,7 @@ describe('Dash Table Queries', () => { ]); }); - describe('in', () => { + describe('like', () => { processCases(c.syntaxer, [ { name: '0yyy in "0"', query: `${c.hideOperand ? '' : '{a} '}like "0"`, target: { a: '0987' }, valid: true, evaluate: false }, { name: '0yyy in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}like "0987"`, target: { a: '0987' }, valid: true, evaluate: true }, diff --git a/tests/cypress/tests/unit/query_syntactic_tree_test.ts b/tests/cypress/tests/unit/query_syntactic_tree_test.ts index cdd88771b..5c2adba98 100644 --- a/tests/cypress/tests/unit/query_syntactic_tree_test.ts +++ b/tests/cypress/tests/unit/query_syntactic_tree_test.ts @@ -20,21 +20,21 @@ describe('Query Syntax Tree', () => { ]); }); - describe('operands', () => { - it('does not support badly formed operands', () => { + describe('expressions', () => { + it('does not support badly formed lhs', () => { expect(new QuerySyntaxTree(`{myField} eq 0`).isValid).to.equal(true); expect(new QuerySyntaxTree(`{'myField'} eq 0`).isValid).to.equal(true); expect(new QuerySyntaxTree(`{"myField"} eq 0`).isValid).to.equal(true); expect(new QuerySyntaxTree('{`myField`} eq 0').isValid).to.equal(true); expect(new QuerySyntaxTree('{\\{myField\\}} eq 0').isValid).to.equal(true); + expect(new QuerySyntaxTree(`myField eq 0`).isValid).to.equal(true); expect(new QuerySyntaxTree('{{myField}} eq 0').isValid).to.equal(false); expect(new QuerySyntaxTree(`{myField eq 0`).isValid).to.equal(false); expect(new QuerySyntaxTree(`myField} eq 0`).isValid).to.equal(false); - expect(new QuerySyntaxTree(`myField eq 0`).isValid).to.equal(false); expect(new QuerySyntaxTree('{\\\\{myField\\\\}} eq 0').isValid).to.equal(false); }); - it('does not support badly formed expression', () => { + it('does not support badly formed rhs', () => { expect(new QuerySyntaxTree(`{myField} eq 'value'`).isValid).to.equal(true); expect(new QuerySyntaxTree(`{myField} eq "value"`).isValid).to.equal(true); expect(new QuerySyntaxTree('{myField} eq `value`').isValid).to.equal(true); From b276e530ae18071ad260c117517bdf22bd0f8ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 12:30:06 -0400 Subject: [PATCH 26/32] filter out empty column filters --- src/dash-table/components/FilterFactory.tsx | 4 ++-- src/dash-table/syntax-tree/index.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dash-table/components/FilterFactory.tsx b/src/dash-table/components/FilterFactory.tsx index 6ebbdf80b..4ba55b9b5 100644 --- a/src/dash-table/components/FilterFactory.tsx +++ b/src/dash-table/components/FilterFactory.tsx @@ -64,8 +64,8 @@ export default class FilterFactory { const globalFilter = getMultiColumnQueryString(asts); const rawGlobalFilter = R.map( - ast => ast.query || '', - R.filter(ast => Boolean(ast), asts) + ast => ast.query, + R.filter(ast => ast && ast.isValid && ast.query !== '', asts) ).join(' && '); setFilter(globalFilter, rawGlobalFilter); diff --git a/src/dash-table/syntax-tree/index.ts b/src/dash-table/syntax-tree/index.ts index c650d9237..46d307751 100644 --- a/src/dash-table/syntax-tree/index.ts +++ b/src/dash-table/syntax-tree/index.ts @@ -12,7 +12,7 @@ export const getMultiColumnQueryString = ( asts: SingleColumnSyntaxTree[] ) => R.map( ast => ast.toQueryString(), - R.filter(ast => ast && ast.isValid, asts) + R.filter(ast => ast && ast.isValid && ast.query !== '', asts) ).join(' && '); export const getSingleColumnMap = ( From e51f7ecb07744a424c248ccc78540f9d69b4be3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 12:31:31 -0400 Subject: [PATCH 27/32] rework --- src/core/syntax-tree/lexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/syntax-tree/lexer.ts b/src/core/syntax-tree/lexer.ts index 2912f8258..170ca9350 100644 --- a/src/core/syntax-tree/lexer.ts +++ b/src/core/syntax-tree/lexer.ts @@ -45,7 +45,7 @@ export default function lexer(lexicon: Lexicon, query: string): ILexerResult { const [terminalPrevious, last] = [ undefined, undefined, - ...result.slice(-2) + ...result ].slice(-2); const terminal: boolean = !last || From 350b5e47fe74fe22ad690ab3918ed1fa8b941abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 13:20:24 -0400 Subject: [PATCH 28/32] revert filterfactory change --- src/dash-table/components/FilterFactory.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dash-table/components/FilterFactory.tsx b/src/dash-table/components/FilterFactory.tsx index 4ba55b9b5..6ebbdf80b 100644 --- a/src/dash-table/components/FilterFactory.tsx +++ b/src/dash-table/components/FilterFactory.tsx @@ -64,8 +64,8 @@ export default class FilterFactory { const globalFilter = getMultiColumnQueryString(asts); const rawGlobalFilter = R.map( - ast => ast.query, - R.filter(ast => ast && ast.isValid && ast.query !== '', asts) + ast => ast.query || '', + R.filter(ast => Boolean(ast), asts) ).join(' && '); setFilter(globalFilter, rawGlobalFilter); From d9055511a629b50d2c3185194b8508b469898895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 14:43:25 -0400 Subject: [PATCH 29/32] - reowrk `contains` and `eq` logic --- .../syntax-tree/lexeme/relational.ts | 23 +++++---- .../tests/unit/dash_table_queries_test.ts | 47 +++++++++++++++++-- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/dash-table/syntax-tree/lexeme/relational.ts b/src/dash-table/syntax-tree/lexeme/relational.ts index ad975b906..bbf5a8e6c 100644 --- a/src/dash-table/syntax-tree/lexeme/relational.ts +++ b/src/dash-table/syntax-tree/lexeme/relational.ts @@ -50,23 +50,22 @@ const LEXEME_BASE = { }; export const contains: IUnboundedLexeme = R.merge({ - evaluate: relationalEvaluator(([op, exp]) => { - if (R.type(exp) !== 'String') { - return false; - } - - return !R.isNil(op) && R.toString(op).indexOf(exp) !== -1; - }), + evaluate: relationalEvaluator(([op, exp]) => + !R.isNil(exp) && + !R.isNil(op) && + (R.type(exp) === 'String' || R.type(op) === 'String') && + op.toString().indexOf(exp.toString()) !== -1 + ), subType: RelationalOperator.Contains, regexp: /^(contains)/i }, LEXEME_BASE); export const equal: IUnboundedLexeme = R.merge({ - evaluate: relationalEvaluator(([op, exp]) => ( - isNumeric(op) && - isNumeric(exp) && - +op === +exp - ) || op === exp), + evaluate: relationalEvaluator(([op, exp]) => + (isNumeric(op) && isNumeric(exp)) ? + +op === +exp : + op === exp + ), subType: RelationalOperator.Equal, regexp: /^(=|eq)/i }, LEXEME_BASE); diff --git a/tests/cypress/tests/unit/dash_table_queries_test.ts b/tests/cypress/tests/unit/dash_table_queries_test.ts index 6c4bebc31..36a02e624 100644 --- a/tests/cypress/tests/unit/dash_table_queries_test.ts +++ b/tests/cypress/tests/unit/dash_table_queries_test.ts @@ -40,7 +40,9 @@ describe('Dash Table Queries', () => { { name: 'compares "1" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: '1' }, valid: true, evaluate: true }, { name: 'compares "1.0" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: '1.0' }, valid: true, evaluate: true }, { name: 'compares "1.0" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: '1.0' }, valid: true, evaluate: true }, - { name: 'compares "1.1" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: '1.1' }, valid: true, evaluate: false }, + { name: 'compares "1" to 1.0', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: '1' }, valid: true, evaluate: true }, + { name: 'compares "1" to "1.0"', query: `${c.hideOperand ? '' : '{a} '}eq "1.0"`, target: { a: '1' }, valid: true, evaluate: true }, + { name: 'compares "1.1" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1.0`, target: { a: '1.1' }, valid: true, evaluate: false }, { name: 'compares "1.1" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: '1.1' }, valid: true, evaluate: false }, { name: 'compares "1" to 0x1', query: `${c.hideOperand ? '' : '{a} '}eq 0x1`, target: { a: '1' }, valid: true, evaluate: true }, { name: 'compares "1" to "0x1"', query: `${c.hideOperand ? '' : '{a} '}eq "0x1"`, target: { a: '1' }, valid: true, evaluate: true }, @@ -53,24 +55,44 @@ describe('Dash Table Queries', () => { { name: 'compares 1 to "0x1"', query: `${c.hideOperand ? '' : '{a} '}eq "0x1"`, target: { a: 1 }, valid: true, evaluate: true }, { name: 'compares "x1" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: 'x1' }, valid: true, evaluate: false }, - { name: 'compares "x1" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: 'x1' }, valid: true, evaluate: false } + { name: 'compares "x1" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: 'x1' }, valid: true, evaluate: false }, + + { name: 'compares "1x" to 1', query: `${c.hideOperand ? '' : '{a} '}eq 1`, target: { a: '1x' }, valid: true, evaluate: false }, + { name: 'compares "1x" to "1"', query: `${c.hideOperand ? '' : '{a} '}eq "1"`, target: { a: '1x' }, valid: true, evaluate: false }, + { name: 'compares 1 to 1x', query: `${c.hideOperand ? '' : '{a} '}eq 1x`, target: { a: 1 }, valid: true, evaluate: false }, + { name: 'compares 1 to "1x"', query: `${c.hideOperand ? '' : '{a} '}eq "1x"`, target: { a: 1 }, valid: true, evaluate: false }, + { name: 'compares "1" to 1x', query: `${c.hideOperand ? '' : '{a} '}eq 1x`, target: { a: '1' }, valid: true, evaluate: false }, + { name: 'compares "1" to "1x"', query: `${c.hideOperand ? '' : '{a} '}eq "1x"`, target: { a: '1' }, valid: true, evaluate: false }, + + { name: 'compares "1" to " 1 "', query: `${c.hideOperand ? '' : '{a} '}eq " 1 "`, target: { a: '1' }, valid: true, evaluate: true }, + { name: 'compares "1" to "\t1\t"', query: `${c.hideOperand ? '' : '{a} '}eq "\t1\t"`, target: { a: '1' }, valid: true, evaluate: true }, + { name: 'compares "1" to "\r\n1\r\n"', query: `${c.hideOperand ? '' : '{a} '}eq "\r\n1\r\n"`, target: { a: '1' }, valid: true, evaluate: true }, + { name: 'compares "\t1\t" to "\r\n1\r\n"', query: `${c.hideOperand ? '' : '{a} '}eq "\r\n1\r\n"`, target: { a: '\t1\t' }, valid: true, evaluate: true }, + { name: 'compares "\r\n1\r\n" to "\t1\t"', query: `${c.hideOperand ? '' : '{a} '}eq "\t1\t"`, target: { a: '\r\n1\r\n' }, valid: true, evaluate: true } ]); }); describe('contains', () => { processCases(c.syntaxer, [ - { name: 'cannot compare "11" to 1', query: `${c.hideOperand ? '' : '{a} '}contains 1`, target: { a: '11' }, valid: true, evaluate: false }, + { name: 'cannot compare "11" to 1', query: `${c.hideOperand ? '' : '{a} '}contains 1`, target: { a: '11' }, valid: true, evaluate: true }, { name: 'cannot compare 11 to 1', query: `${c.hideOperand ? '' : '{a} '}contains 1`, target: { a: 11 }, valid: true, evaluate: false }, { name: 'compares "11" to "1"', query: `${c.hideOperand ? '' : '{a} '}contains "1"`, target: { a: '11' }, valid: true, evaluate: true }, { name: 'compares 11 to "1"', query: `${c.hideOperand ? '' : '{a} '}contains "1"`, target: { a: 11 }, valid: true, evaluate: true }, { name: 'compares "1" to "1.0"', query: `${c.hideOperand ? '' : '{a} '}contains "1.0"`, target: { a: '1' }, valid: true, evaluate: false }, - { name: 'compares 1 to "1.0"', query: `${c.hideOperand ? '' : '{a} '}contains "1.0"`, target: { a: 1 }, valid: true, evaluate: false } + { name: 'compares 1 to "1.0"', query: `${c.hideOperand ? '' : '{a} '}contains "1.0"`, target: { a: 1 }, valid: true, evaluate: false }, + + { name: 'compares "abc" to "b"', query: `${c.hideOperand ? '' : '{a} '}contains "b"`, target: { a: 'abc' }, valid: true, evaluate: true }, + { name: 'compares "abc" to " b"', query: `${c.hideOperand ? '' : '{a} '}contains " b"`, target: { a: 'abc' }, valid: true, evaluate: false }, + { name: 'compares "abc" to "b "', query: `${c.hideOperand ? '' : '{a} '}contains "b "`, target: { a: 'abc' }, valid: true, evaluate: false }, + { name: 'compares "abc" to " b"', query: `${c.hideOperand ? '' : '{a} '}contains " b"`, target: { a: 'a bc' }, valid: true, evaluate: true }, + { name: 'compares "abc" to "b "', query: `${c.hideOperand ? '' : '{a} '}contains "b "`, target: { a: 'ab c' }, valid: true, evaluate: true } ]); }); describe('like', () => { processCases(c.syntaxer, [ { name: '0yyy in "0"', query: `${c.hideOperand ? '' : '{a} '}like "0"`, target: { a: '0987' }, valid: true, evaluate: false }, + { name: '0 in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}like "0987"`, target: { a: '0' }, valid: true, evaluate: false }, { name: '0yyy in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}like "0987"`, target: { a: '0987' }, valid: true, evaluate: true }, { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2006"`, target: { a: '2005' }, valid: true, evaluate: false }, { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005' }, valid: true, evaluate: true }, @@ -93,7 +115,22 @@ describe('Dash Table Queries', () => { { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx000', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx111', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000111`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false } + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx111', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000111`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, + + { name: 'yyyy-01 in yyyy-02', query: `${c.hideOperand ? '' : '{a} '}like 2005-02`, target: { a: '2005-01' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-01 in yyyy-mm-02', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-02`, target: { a: '2005-01-01' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd 00 in yyyy-mm-dd 01', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T01`, target: { a: '2005-01-01 00' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:00 in yyyy-mm-dd hh:01', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T00:01`, target: { a: '2005-01-01 00:00' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:mm:00 in yyyy-mm-dd hh:mm:01', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T00:00:01`, target: { a: '2005-01-01 00:00:00' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:mm:ss.000 in yyyy-mm-dd hh:mm:ss.001', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T00:00:00.001`, target: { a: '2005-01-01 00:00:00.000' }, valid: true, evaluate: false }, + + { name: '20yy in "yy"', query: `${c.hideOperand ? '' : '{a} '}like "19"`, target: { a: '2019' }, valid: true, evaluate: true }, + { name: '19yy in "yy"', query: `${c.hideOperand ? '' : '{a} '}like "19"`, target: { a: '1919' }, valid: true, evaluate: false }, + { name: 'yy in "19yy"', query: `${c.hideOperand ? '' : '{a} '}like "1999"`, target: { a: '99' }, valid: true, evaluate: true }, + { name: 'yy in "20yy"', query: `${c.hideOperand ? '' : '{a} '}like "2099"`, target: { a: '99' }, valid: true, evaluate: false }, + { name: 'yy in yy', query: `${c.hideOperand ? '' : '{a} '}like "79"`, target: { a: '79' }, valid: true, evaluate: true }, + { name: 'yy in YY"', query: `${c.hideOperand ? '' : '{a} '}like "79"`, target: { a: '78' }, valid: true, evaluate: false } + ]); }); }); From 72116ea2269da82415297f19fabe721afcba4828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 15:33:01 -0400 Subject: [PATCH 30/32] - operand / expression regression --- src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts b/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts index 99ca389a7..15c044023 100644 --- a/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts +++ b/src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts @@ -43,7 +43,7 @@ export default class MultiColumnsSyntaxTree extends SyntaxTree { const fields = R.map( (item: ILexemeResult) => item.value, R.filter( - i => i.lexeme.type === LexemeType.Expression, + i => i.lexeme.type === LexemeType.Expression && i.lexeme.subType === 'field', this.lexerResult.lexemes ) ); From 7971d765d2bce6762f0ba866a636a347b6a14324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 15:51:48 -0400 Subject: [PATCH 31/32] rename `like` relational operator to `dateStartsWith` --- .../syntax-tree/SingleColumnSyntaxTree.ts | 2 +- .../syntax-tree/lexeme/relational.ts | 8 +- src/dash-table/syntax-tree/lexicon/column.ts | 4 +- .../syntax-tree/lexicon/columnMulti.ts | 4 +- src/dash-table/syntax-tree/lexicon/query.ts | 4 +- .../tests/unit/dash_table_queries_test.ts | 80 +++++++++---------- 6 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts index 01707c725..6651f3bc0 100644 --- a/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts +++ b/src/dash-table/syntax-tree/SingleColumnSyntaxTree.ts @@ -16,7 +16,7 @@ function getDefaultRelationalOperator(type: ColumnType = ColumnType.Any): Relati case ColumnType.Text: return RelationalOperator.Contains; case ColumnType.Datetime: - return RelationalOperator.LikeDate; + return RelationalOperator.DateStartsWith; case ColumnType.Numeric: return RelationalOperator.Equal; } diff --git a/src/dash-table/syntax-tree/lexeme/relational.ts b/src/dash-table/syntax-tree/lexeme/relational.ts index bbf5a8e6c..1c9a92ea9 100644 --- a/src/dash-table/syntax-tree/lexeme/relational.ts +++ b/src/dash-table/syntax-tree/lexeme/relational.ts @@ -34,12 +34,12 @@ function relationalEvaluator( export enum RelationalOperator { Contains = 'contains', + DateStartsWith = 'datestartswith', Equal = '=', GreaterOrEqual = '>=', GreaterThan = '>', LessOrEqual = '<=', LessThan = '<', - LikeDate = 'like', NotEqual = '!=' } @@ -86,7 +86,7 @@ const DATE_OPTIONS: IDateValidation = { allow_YY: true }; -export const likeDate: IUnboundedLexeme = R.merge({ +export const dateStartsWith: IUnboundedLexeme = R.merge({ evaluate: relationalEvaluator(([op, exp]) => { const normalizedOp = normalizeDate(op, DATE_OPTIONS); const normalizedExp = normalizeDate(exp, DATE_OPTIONS); @@ -96,8 +96,8 @@ export const likeDate: IUnboundedLexeme = R.merge({ // IE11 does not support `startsWith` normalizedOp.indexOf(normalizedExp) === 0; }), - subType: RelationalOperator.LikeDate, - regexp: /^(like)/i + subType: RelationalOperator.DateStartsWith, + regexp: /^(datestartswith)/i }, LEXEME_BASE); export const lessOrEqual: IUnboundedLexeme = R.merge({ diff --git a/src/dash-table/syntax-tree/lexicon/column.ts b/src/dash-table/syntax-tree/lexicon/column.ts index ed01851e8..b73c85231 100644 --- a/src/dash-table/syntax-tree/lexicon/column.ts +++ b/src/dash-table/syntax-tree/lexicon/column.ts @@ -5,12 +5,12 @@ import { } from '../lexeme/expression'; import { contains, + dateStartsWith, equal, greaterOrEqual, greaterThan, lessOrEqual, lessThan, - likeDate, notEqual } from '../lexeme/relational'; import { @@ -33,12 +33,12 @@ import { ILexeme } from 'core/syntax-tree/lexicon'; const lexicon: ILexeme[] = [ ...[contains, + dateStartsWith, equal, greaterOrEqual, greaterThan, lessOrEqual, lessThan, - likeDate, notEqual ].map(op => ({ ...op, diff --git a/src/dash-table/syntax-tree/lexicon/columnMulti.ts b/src/dash-table/syntax-tree/lexicon/columnMulti.ts index 74fd87206..43406f7df 100644 --- a/src/dash-table/syntax-tree/lexicon/columnMulti.ts +++ b/src/dash-table/syntax-tree/lexicon/columnMulti.ts @@ -10,12 +10,12 @@ import { } from '../lexeme/logical'; import { contains, + dateStartsWith, equal, greaterOrEqual, greaterThan, lessOrEqual, lessThan, - likeDate, notEqual } from '../lexeme/relational'; import { @@ -44,12 +44,12 @@ const lexicon: ILexeme[] = [ terminal: false }, ...[contains, + dateStartsWith, equal, greaterOrEqual, greaterThan, lessOrEqual, lessThan, - likeDate, notEqual ].map(op => ({ ...op, diff --git a/src/dash-table/syntax-tree/lexicon/query.ts b/src/dash-table/syntax-tree/lexicon/query.ts index 6049caf0a..66b34be2d 100644 --- a/src/dash-table/syntax-tree/lexicon/query.ts +++ b/src/dash-table/syntax-tree/lexicon/query.ts @@ -18,12 +18,12 @@ import { } from '../lexeme/logical'; import { contains, + dateStartsWith, equal, greaterOrEqual, greaterThan, lessOrEqual, lessThan, - likeDate, notEqual } from '../lexeme/relational'; import { @@ -77,12 +77,12 @@ const lexicon: ILexeme[] = [ terminal: false }, ...[contains, + dateStartsWith, equal, greaterOrEqual, greaterThan, lessOrEqual, lessThan, - likeDate, notEqual ].map(op => ({ ...op, diff --git a/tests/cypress/tests/unit/dash_table_queries_test.ts b/tests/cypress/tests/unit/dash_table_queries_test.ts index 36a02e624..d811e5731 100644 --- a/tests/cypress/tests/unit/dash_table_queries_test.ts +++ b/tests/cypress/tests/unit/dash_table_queries_test.ts @@ -89,47 +89,47 @@ describe('Dash Table Queries', () => { ]); }); - describe('like', () => { + describe('datestartswith', () => { processCases(c.syntaxer, [ - { name: '0yyy in "0"', query: `${c.hideOperand ? '' : '{a} '}like "0"`, target: { a: '0987' }, valid: true, evaluate: false }, - { name: '0 in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}like "0987"`, target: { a: '0' }, valid: true, evaluate: false }, - { name: '0yyy in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}like "0987"`, target: { a: '0987' }, valid: true, evaluate: true }, - { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2006"`, target: { a: '2005' }, valid: true, evaluate: false }, - { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005' }, valid: true, evaluate: true }, - { name: 'yyyy in yyyy', query: `${c.hideOperand ? '' : '{a} '}like 2005`, target: { a: '2005' }, valid: true, evaluate: false }, - { name: 'yyyy-mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}like "2005"`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, - { name: 'yyyy in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005' }, valid: true, evaluate: false }, - { name: 'yyyy-mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm in "yyyy-mm"', query: `${c.hideOperand ? '' : '{a} '}like "2005-01"`, target: { a: '2005-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}like 2005-01`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx000', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, - { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx111', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T10:00:00.000111`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, - - { name: 'yyyy-01 in yyyy-02', query: `${c.hideOperand ? '' : '{a} '}like 2005-02`, target: { a: '2005-01' }, valid: true, evaluate: false }, - { name: 'yyyy-mm-01 in yyyy-mm-02', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-02`, target: { a: '2005-01-01' }, valid: true, evaluate: false }, - { name: 'yyyy-mm-dd 00 in yyyy-mm-dd 01', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T01`, target: { a: '2005-01-01 00' }, valid: true, evaluate: false }, - { name: 'yyyy-mm-dd hh:00 in yyyy-mm-dd hh:01', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T00:01`, target: { a: '2005-01-01 00:00' }, valid: true, evaluate: false }, - { name: 'yyyy-mm-dd hh:mm:00 in yyyy-mm-dd hh:mm:01', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T00:00:01`, target: { a: '2005-01-01 00:00:00' }, valid: true, evaluate: false }, - { name: 'yyyy-mm-dd hh:mm:ss.000 in yyyy-mm-dd hh:mm:ss.001', query: `${c.hideOperand ? '' : '{a} '}like 2005-01-01T00:00:00.001`, target: { a: '2005-01-01 00:00:00.000' }, valid: true, evaluate: false }, - - { name: '20yy in "yy"', query: `${c.hideOperand ? '' : '{a} '}like "19"`, target: { a: '2019' }, valid: true, evaluate: true }, - { name: '19yy in "yy"', query: `${c.hideOperand ? '' : '{a} '}like "19"`, target: { a: '1919' }, valid: true, evaluate: false }, - { name: 'yy in "19yy"', query: `${c.hideOperand ? '' : '{a} '}like "1999"`, target: { a: '99' }, valid: true, evaluate: true }, - { name: 'yy in "20yy"', query: `${c.hideOperand ? '' : '{a} '}like "2099"`, target: { a: '99' }, valid: true, evaluate: false }, - { name: 'yy in yy', query: `${c.hideOperand ? '' : '{a} '}like "79"`, target: { a: '79' }, valid: true, evaluate: true }, - { name: 'yy in YY"', query: `${c.hideOperand ? '' : '{a} '}like "79"`, target: { a: '78' }, valid: true, evaluate: false } + { name: '0yyy in "0"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "0"`, target: { a: '0987' }, valid: true, evaluate: false }, + { name: '0 in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "0987"`, target: { a: '0' }, valid: true, evaluate: false }, + { name: '0yyy in "0yyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "0987"`, target: { a: '0987' }, valid: true, evaluate: true }, + { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2006"`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005"`, target: { a: '2005' }, valid: true, evaluate: true }, + { name: 'yyyy in yyyy', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy-mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005"`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005"`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005"`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005"`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005"`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005"`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in "yyyy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005"`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, + { name: 'yyyy in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01`, target: { a: '2005' }, valid: true, evaluate: false }, + { name: 'yyyy-mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm in "yyyy-mm"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2005-01"`, target: { a: '2005-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01`, target: { a: '2005-01-01' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01`, target: { a: '2005-01-01T10' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01`, target: { a: '2005-01-01T10:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01`, target: { a: '2005-01-01 10:00:00' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxxxxxxxx in yyyy-mm', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01`, target: { a: '2005-01-01 10:00:00.000000000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01-01T10:00:00.000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: true }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx000', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01-01T10:00:00.000000`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:mm:ss.xxx in yyyy-mm-ddThh:mm:ss.xxx111', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01-01T10:00:00.000111`, target: { a: '2005-01-01 10:00:00.000' }, valid: true, evaluate: false }, + + { name: 'yyyy-01 in yyyy-02', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-02`, target: { a: '2005-01' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-01 in yyyy-mm-02', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01-02`, target: { a: '2005-01-01' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd 00 in yyyy-mm-dd 01', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01-01T01`, target: { a: '2005-01-01 00' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:00 in yyyy-mm-dd hh:01', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01-01T00:01`, target: { a: '2005-01-01 00:00' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:mm:00 in yyyy-mm-dd hh:mm:01', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01-01T00:00:01`, target: { a: '2005-01-01 00:00:00' }, valid: true, evaluate: false }, + { name: 'yyyy-mm-dd hh:mm:ss.000 in yyyy-mm-dd hh:mm:ss.001', query: `${c.hideOperand ? '' : '{a} '}datestartswith 2005-01-01T00:00:00.001`, target: { a: '2005-01-01 00:00:00.000' }, valid: true, evaluate: false }, + + { name: '20yy in "yy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "19"`, target: { a: '2019' }, valid: true, evaluate: true }, + { name: '19yy in "yy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "19"`, target: { a: '1919' }, valid: true, evaluate: false }, + { name: 'yy in "19yy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "1999"`, target: { a: '99' }, valid: true, evaluate: true }, + { name: 'yy in "20yy"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "2099"`, target: { a: '99' }, valid: true, evaluate: false }, + { name: 'yy in yy', query: `${c.hideOperand ? '' : '{a} '}datestartswith "79"`, target: { a: '79' }, valid: true, evaluate: true }, + { name: 'yy in YY"', query: `${c.hideOperand ? '' : '{a} '}datestartswith "79"`, target: { a: '78' }, valid: true, evaluate: false } ]); }); From 9fda146d4796faf1f499f73e174946d05e7a70a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andre=CC=81=20Rivet?= Date: Thu, 25 Apr 2019 16:18:32 -0400 Subject: [PATCH 32/32] update changelog --- CHANGELOG.md | 12 ++++++++---- src/dash-table/dash/DataTable.js | 3 --- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd4b5b81f..572d99c3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,18 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added -[#397](https://github.com/plotly/dash-table/pull/397) +[#397](https://github.com/plotly/dash-table/pull/397), [#410](https://github.com/plotly/dash-table/pull/410) - Improve filtering syntax and capabilities - new field syntax `{myField}` - short form by-column filter - - implicit column and `eq` operator (e.g `"value"`) - - implicit column (e.g `ne "value"`) - - explicit form (e.g `{field} ne "value"`) + - implicit column and default operator based on column type + - Text and Any columns default to `contains` + - Numeric columns default to `eq` + - Date columns default to `datestartswith` + - implicit column (e.g `ne "value"` becomes `{my-column} ne "value"`) - new `contains` relational operator for strings + - new `datestartswith` relational operator for dates + - new `eq` behavior (will attempt to convert and compare numeric values if possible) - new readonly `derived_filter_structure` prop exposing the query structure in a programmatically friendlier way ### Changed diff --git a/src/dash-table/dash/DataTable.js b/src/dash-table/dash/DataTable.js index 52119b881..e140777da 100644 --- a/src/dash-table/dash/DataTable.js +++ b/src/dash-table/dash/DataTable.js @@ -992,18 +992,15 @@ export const propTypes = { * - 'relational-operator' * - 'unary-operator' * - 'expression' - * - 'operand' * - subType (string; optional) * - 'open-block': '()' * - 'logical-operator': '&&', '||' * - 'relational-operator': '=', '>=', '>', '<=', '<', '!=', 'contains' * - 'unary-operator': '!', 'is bool', 'is even', 'is nil', 'is num', 'is object', 'is odd', 'is prime', 'is str' * - 'expression': 'value', 'field' - * - 'operand': 'field' * - value (any) * - 'expression, value': passed value * - 'expression, field': the field/prop name - * - 'operand, field': the field/prop name * * - block (nested query structure; optional) * - left (nested query structure; optional)