Improve filtering type awareness (#410)

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2
 
 jobs:
-    "test":
+    "server-test":
         docker:
             - image: circleci/python:3.6.7-node-browsers
             - image: cypress/base:10
@@ -44,7 +44,73 @@ jobs:
                 name: Run tests
                 command: |
                     . venv/bin/activate
-                    npm run test
+                    npm run test.server
+
+
+    "standalone-test":
+        docker:
+            - image: circleci/python:3.6.7-node-browsers
+            - image: cypress/base:10
+
+        steps:
+            - checkout
+            - restore_cache:
+                key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
+            - run:
+                name: Install npm packages
+                command: npm install
+            - run:
+                name: Cypress Install
+                command: |
+                    $(npm bin)/cypress install
+
+            - save_cache:
+                key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
+                paths:
+                    - node_modules
+                    - /home/circleci/.cache/Cypress
+
+            - run:
+                name: Run tests
+                command: npm run test.standalone
+
+
+    "unit-test":
+        docker:
+            - image: circleci/python:3.6.7-node-browsers
+            - image: cypress/base:10
+
+        steps:
+            - checkout
+            - restore_cache:
+                key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
+            - run:
+                name: Install npm packages
+                command: npm install
+            - run:
+                name: Cypress Install
+                command: |
+                    $(npm bin)/cypress install
+
+            - save_cache:
+                key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
+                paths:
+                    - node_modules
+                    - /home/circleci/.cache/Cypress
+
+            - run:
+                name: Install requirements
+                command: |
+                    sudo pip install --upgrade virtualenv
+                    python -m venv venv || virtualenv venv
+                    . venv/bin/activate
+                    pip install -r requirements.txt --quiet
+
+            - run:
+                name: Run tests
+                command: |
+                    . venv/bin/activate
+                    npm run test.unit
 
 
     "visual-test":
@@ -170,5 +236,7 @@ workflows:
         jobs:
             - "python-3.6"
             - "node"
-            - "test"
+            - "server-test"
+            - "standalone-test"
+            - "unit-test"
             - "visual-test"

CHANGELOG.md

@@ -4,14 +4,18 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 ### Added
-[#397](https://github.com/plotly/dash-table/pull/397)
+[#397](https://github.com/plotly/dash-table/pull/397), [#410](https://github.com/plotly/dash-table/pull/410)
 - Improve filtering syntax and capabilities
     - new field syntax `{myField}`
     - short form by-column filter
-        - implicit column and `eq` operator (e.g `"value"`)
-        - implicit column (e.g `ne "value"`)
-        - explicit form (e.g `{field} ne "value"`)
+        - implicit column and default operator based on column type
+            - Text and Any columns default to `contains`
+            - Numeric columns default to `eq`
+            - Date columns default to `datestartswith`
+        - implicit column (e.g `ne "value"` becomes `{my-column} ne "value"`)
     - new `contains` relational operator for strings
+    - new `datestartswith` relational operator for dates
+    - new `eq` behavior (will attempt to convert and compare numeric values if possible)
     - new readonly `derived_filter_structure` prop exposing the query structure in a programmatically friendlier way
 
 ### Changed

package.json

@@ -5,7 +5,9 @@
   "main": "dash_table/bundle.js",
   "scripts": {
     "preprivate::opentests": "run-s private::wait*",
-    "preprivate::runtests": "run-s private::wait*",
+    "preprivate::test.server": "run-s private::wait_dash*",
+    "preprivate::test.standalone": "run-s private::wait_js",
+    "pretest.standalone": "run-s private::build:js-test",
     "private::build": "node --max_old_space_size=4096 node_modules/webpack/bin/webpack --display-reasons --bail",
     "private::build:js": "run-s \"private::build -- --mode production\"",
     "private::build:js-dev": "run-s \"private::build -- --mode development\"",
@@ -26,15 +28,16 @@
     "private::wait_dash8083": "wait-on http://localhost:8083",
     "private::wait_js": "wait-on http://localhost:8080",
     "private::opentests": "cypress open",
-    "private::runtests:python": "python -m unittest tests/unit/format_test.py",
-    "private::runtests:unit": "cypress run --browser chrome --spec 'tests/cypress/tests/unit/**/*'",
-    "private::runtests:standalone": "cypress run --browser chrome --spec 'tests/cypress/tests/standalone/**/*'",
-    "private::runtests:server": "cypress run --browser chrome --spec 'tests/cypress/tests/server/**/*'",
-    "private::runtests": "run-s private::runtests:python private::runtests:unit private::runtests:standalone private::runtests:server",
+    "private::test.python": "python -m unittest tests/unit/format_test.py",
+    "private::test.unit": "cypress run --browser chrome --spec 'tests/cypress/tests/unit/**/*'",
+    "private::test.server": "cypress run --browser chrome --spec 'tests/cypress/tests/server/**/*'",
+    "private::test.standalone": "cypress run --browser chrome --spec 'tests/cypress/tests/standalone/**/*'",
     "build.watch": "webpack-dev-server --content-base dash_table --mode development",
     "build": "run-s private::build:js private::build:py",
     "lint": "run-s private::lint:*",
-    "test": "run-p --race private::host* private::runtests",
+    "test.server": "run-p --race private::host* private::test.server",
+    "test.standalone": "run-p --race private::host_js private::test.standalone",
+    "test.unit": "run-s private::test.python private::test.unit",
     "test.visual": "build-storybook && percy-storybook",
     "test.visual-local": "build-storybook",
     "test.watch": "run-p --race \"private::build:js-test-watch\" --race private::host* private::opentests"
@@ -54,7 +57,7 @@
     "@storybook/react": "^5.0.5",
     "@types/d3-format": "^1.3.1",
     "@types/papaparse": "^4.5.9",
-    "@types/ramda": "^0.26.5",
+    "@types/ramda": "^0.26.6",
     "@types/react": "^16.8.8",
     "@types/react-dom": "^16.8.3",
     "@types/react-select": "^1.3.4",
@@ -82,7 +85,7 @@
     "style-loader": "^0.23.1",
     "ts-loader": "^5.3.3",
     "tslint": "^5.14.0",
-    "typescript": "^3.3.4000",
+    "typescript": "^3.4.3",
     "wait-on": "^3.2.0",
     "webpack": "^4.29.6",
     "webpack-cli": "^3.3.0",

src/core/syntax-tree/lexer.ts

@@ -42,14 +42,20 @@ export default function lexer(lexicon: Lexicon, query: string): ILexerResult {
         query = query.substring(value.length);
     }
 
-    const last = result.slice(-1)[0];
-
-    const terminal: boolean = last && (typeof last.lexeme.terminal === 'function' ?
-        last.lexeme.terminal(result, last) :
-        last.lexeme.terminal);
+    const [terminalPrevious, last] = [
+        undefined,
+        undefined,
+        ...result
+    ].slice(-2);
+
+    const terminal: boolean = !last ||
+        (typeof last.lexeme.terminal === 'function' ?
+            last.lexeme.terminal(result, terminalPrevious) :
+            last.lexeme.terminal
+        );
 
     return {
         lexemes: result,
-        valid: !last || terminal
+        valid: terminal
     };
 }

src/core/syntax-tree/lexicon.ts

@@ -7,8 +7,7 @@ export enum LexemeType {
     LogicalOperator = 'logical-operator',
     RelationalOperator = 'relational-operator',
     UnaryOperator = 'unary-operator',
-    Expression = 'expression',
-    Operand = 'operand'
+    Expression = 'expression'
 }
 
 export interface IUnboundedLexeme {
@@ -25,8 +24,8 @@ export interface IUnboundedLexeme {
 }
 
 export interface ILexeme extends IUnboundedLexeme {
-    terminal: boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean);
-    if: (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult) => boolean);
+    terminal: boolean | ((lexemes: ILexemeResult[], previous: ILexemeResult | undefined) => boolean);
+    if: (string | undefined)[] | ((lexemes: ILexemeResult[], previous: ILexemeResult | undefined) => boolean);
 }
 
 export function boundLexeme(lexeme: IUnboundedLexeme) {

src/core/type/index.ts

@@ -0,0 +1,2 @@
+export type RequiredPluck<T, R extends keyof T> = { [r in R]: T[r] };
+export type OptionalPluck<T, R extends keyof T> = { [r in R]?: T[r] };

src/dash-table/components/FilterFactory.tsx

@@ -48,14 +48,14 @@ export default class FilterFactory {
 
     }
 
-    private onChange = (columnId: ColumnId, setFilter: SetFilter, ev: any) => {
-        Logger.debug('Filter -- onChange', columnId, ev.target.value && ev.target.value.trim());
+    private onChange = (column: IVisibleColumn, setFilter: SetFilter, ev: any) => {
+        Logger.debug('Filter -- onChange', column.id, ev.target.value && ev.target.value.trim());
 
         const value = ev.target.value.trim();
-        const safeColumnId = columnId.toString();
+        const safeColumnId = column.id.toString();
 
         if (value && value.length) {
-            this.ops.set(safeColumnId, new SingleColumnSyntaxTree(safeColumnId, value));
+            this.ops.set(safeColumnId, new SingleColumnSyntaxTree(value, column));
         } else {
             this.ops.delete(safeColumnId);
         }
@@ -65,26 +65,26 @@ export default class FilterFactory {
 
         const rawGlobalFilter = R.map(
             ast => ast.query || '',
-            R.filter(ast => Boolean(ast), asts)
+            R.filter<SingleColumnSyntaxTree>(ast => Boolean(ast), asts)
         ).join(' && ');
 
         setFilter(globalFilter, rawGlobalFilter);
     }
 
-    private getEventHandler = (fn: Function, columnId: ColumnId, setFilter: SetFilter): any => {
+    private getEventHandler = (fn: Function, column: IVisibleColumn, setFilter: SetFilter): any => {
         const fnHandler = (this.handlers.get(fn) || this.handlers.set(fn, new Map()).get(fn));
-        const columnIdHandler = (fnHandler.get(columnId) || fnHandler.set(columnId, new Map()).get(columnId));
+        const columnIdHandler = (fnHandler.get(column.id) || fnHandler.set(column.id, new Map()).get(column.id));
 
         return (
             columnIdHandler.get(setFilter) ||
-            (columnIdHandler.set(setFilter, fn.bind(this, columnId, setFilter)).get(setFilter))
+            (columnIdHandler.set(setFilter, fn.bind(this, column, setFilter)).get(setFilter))
         );
     }
 
-    private updateOps = memoizeOne((query: string) => {
+    private updateOps = memoizeOne((query: string, columns: IVisibleColumn[]) => {
         const multiQuery = new MultiColumnsSyntaxTree(query);
 
-        const newOps = getSingleColumnMap(multiQuery);
+        const newOps = getSingleColumnMap(multiQuery, columns);
         if (!newOps) {
             return;
         }
@@ -109,15 +109,15 @@ export default class FilterFactory {
     });
 
     private filter = memoizerCache<[ColumnId, number]>()((
-        column: ColumnId,
+        column: IVisibleColumn,
         index: number,
         ast: SingleColumnSyntaxTree | undefined,
         setFilter: SetFilter
     ) => {
         return (<ColumnFilter
             key={`column-${index}`}
             classes={`dash-filter column-${index}`}
-            columnId={column}
+            columnId={column.id}
             isValid={!ast || ast.isValid}
             setFilter={this.getEventHandler(this.onChange, column, setFilter)}
             value={ast && ast.query}
@@ -143,7 +143,7 @@ export default class FilterFactory {
             return [];
         }
 
-        this.updateOps(filter);
+        this.updateOps(filter, columns);
 
         if (filtering_type === FilteringType.Basic) {
             const filterStyles = this.relevantStyles(
@@ -160,7 +160,7 @@ export default class FilterFactory {
 
             const filters = R.addIndex<IVisibleColumn, JSX.Element>(R.map)((column, index) => {
                 return this.filter.get(column.id, index)(
-                    column.id,
+                    column,
                     index,
                     this.ops.get(column.id.toString()),
                     setFilter

src/dash-table/dash/DataTable.js

@@ -992,18 +992,15 @@ export const propTypes = {
      *   - 'relational-operator'
      *   - 'unary-operator'
      *   - 'expression'
-     *   - 'operand'
      * - subType (string; optional)
      *   - 'open-block': '()'
      *   - 'logical-operator': '&&', '||'
      *   - 'relational-operator': '=', '>=', '>', '<=', '<', '!=', 'contains'
      *   - 'unary-operator': '!', 'is bool', 'is even', 'is nil', 'is num', 'is object', 'is odd', 'is prime', 'is str'
      *   - 'expression': 'value', 'field'
-     *   - 'operand': 'field'
      * - value (any)
      *   - 'expression, value': passed value
      *   - 'expression, field': the field/prop name
-     *   - 'operand, field': the field/prop name
      *
      * - block (nested query structure; optional)
      * - left (nested query structure; optional)

src/dash-table/derived/cell/dropdowns.ts

@@ -96,7 +96,7 @@ class Dropdowns {
             ...(staticDropdown ? [staticDropdown] : []),
             ...R.map(
                 ([cd]) => cd.dropdown,
-                R.filter(
+                R.filter<[IConditionalDropdown, number]>(
                     ([cd, i]) => this.evaluation.get(column.id, i)(
                         this.ast.get(column.id, i)(cd.condition),
                         datum

src/dash-table/derived/cell/wrapperStyles.ts

@@ -16,7 +16,7 @@ function getter(
     return R.addIndex<any, Style[]>(R.map)((datum, index) => R.map(column => {
         const relevantStyles = R.map(
             s => s.style,
-            R.filter(
+            R.filter<IConvertedStyle>(
                 style =>
                     style.matchesColumn(column) &&
                     style.matchesRow(index + offset.rows) &&

src/dash-table/derived/filter/wrapperStyles.ts

@@ -16,7 +16,7 @@ function getter(
     return R.map(column => {
         const relevantStyles = R.map(
             s => s.style,
-            R.filter(
+            R.filter<IConvertedStyle>(
                 style => style.matchesColumn(column),
                 filterStyles
             )

src/dash-table/derived/header/wrapperStyles.ts

@@ -17,7 +17,7 @@ function getter(
     return R.map(idx => R.map(column => {
         const relevantStyles = R.map(
             s => s.style,
-            R.filter(
+            R.filter<IConvertedStyle>(
                 style =>
                     style.matchesColumn(column) &&
                     style.matchesRow(idx),

src/dash-table/syntax-tree/MultiColumnsSyntaxTree.ts

@@ -5,6 +5,7 @@ import { LexemeType } from 'core/syntax-tree/lexicon';
 import { ISyntaxTree } from 'core/syntax-tree/syntaxer';
 
 import columnMultiLexicon from './lexicon/columnMulti';
+import { ILexemeResult } from 'core/syntax-tree/lexer';
 
 export default class MultiColumnsSyntaxTree extends SyntaxTree {
     constructor(query: string) {
@@ -39,7 +40,13 @@ export default class MultiColumnsSyntaxTree extends SyntaxTree {
         return statements;
     }
     private respectsBasicSyntax() {
-        const fields = R.map(item => item.value, R.filter(i => i.lexeme.type === LexemeType.Operand, this.lexerResult.lexemes));
+        const fields = R.map(
+            (item: ILexemeResult) => item.value,
+            R.filter(
+                i => i.lexeme.type === LexemeType.Expression && i.lexeme.subType === 'field',
+                this.lexerResult.lexemes
+            )
+        );
         const uniqueFields = R.uniq(fields);
         return fields.length === uniqueFields.length;
     }