From b547cd84f2588cc9ab564cbc502692d69c627262 Mon Sep 17 00:00:00 2001 From: Patrik Cyvoct Date: Mon, 22 Sep 2025 19:40:13 +0200 Subject: [PATCH 01/14] fix README about grafana 12 Signed-off-by: Patrik Cyvoct --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 901133a..b5207fb 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,25 @@ Quickwit 0.8 is compatible with 0.4.x and 0.5.x versions. You can either download the plugin manually and unzip it into the plugin directory or use the env variable `GF_INSTALL_PLUGINS` to install it. +### 0.5.0 (Latest) for Quickwit 0.8 + Grafana 12.1 + +`GF_INSTALL_PLUGINS` has been deprecated since 12.1. `GF_PLUGINS_PREINSTALL_SYNC` must be used instead + +Run `grafana` container with the env variable: + +```bash +docker run -p 3000:3000 -e GF_PLUGINS_PREINSTALL_SYNC="quickwit-quickwit-datasource@0.5.0@https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.5.0/quickwit-quickwit-datasource-0.5.0.zip" grafana/grafana run +``` + +Or download the plugin manually and start Grafana + +```bash +wget https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.5.0/quickwit-quickwit-datasource-0.5.0.zip +mkdir -p plugins +unzip quickwit-quickwit-datasource-0.5.0.zip -d plugins/quickwit-quickwit-datasource-0.5.0 +docker run -p 3000:3000 -e GF_PATHS_PLUGINS=/data/plugins -v ${PWD}/plugins:/data/plugins grafana/grafana run +``` + ### 0.5.0 (Latest) for Quickwit 0.8 + Grafana 11 Run `grafana` container with the env variable: From e182c0fec4521530617230576e6211a3a7c0e979 Mon Sep 17 00:00:00 2001 From: Ross Sullivan Date: Thu, 11 Sep 2025 16:28:26 +0900 Subject: [PATCH 02/14] feat: Added ability to add quick filters --- .../TermsSettingsEditor.test.tsx | 1 + .../ElasticsearchQueryContext.test.tsx | 1 + .../QueryEditor/ElasticsearchQueryContext.tsx | 6 +- .../QueryEditor/FilterEditor/index.tsx | 149 ++++++++++++++++++ .../QueryEditor/FilterEditor/state/actions.ts | 10 ++ .../QueryEditor/FilterEditor/state/reducer.ts | 100 ++++++++++++ .../MetricEditor.test.tsx | 2 + .../SettingsEditor/index.test.tsx | 2 + src/components/QueryEditor/index.test.tsx | 2 + src/components/QueryEditor/index.tsx | 2 + src/dataquery.gen.ts | 12 ++ src/datasource/base.ts | 79 +++++++--- src/datasource/supplementaryQueries.ts | 1 + src/hooks/useFields.test.tsx | 1 + src/hooks/useFields.ts | 16 +- src/hooks/useNextId.test.tsx | 1 + src/modifyQuery.ts | 15 +- src/queryDef.ts | 15 ++ src/utils/index.ts | 5 + src/utils/uid.ts | 6 + 20 files changed, 402 insertions(+), 24 deletions(-) create mode 100644 src/components/QueryEditor/FilterEditor/index.tsx create mode 100644 src/components/QueryEditor/FilterEditor/state/actions.ts create mode 100644 src/components/QueryEditor/FilterEditor/state/reducer.ts diff --git a/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx b/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx index 8b473ae..f8be73f 100644 --- a/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx +++ b/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx @@ -22,6 +22,7 @@ describe('Terms Settings Editor', () => { query: '', bucketAggs: [termsAgg], metrics: [avg, derivative, topMetrics], + filters: [], }; renderWithESProvider(, { providerProps: { query } }); diff --git a/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx b/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx index bc52130..4db8980 100644 --- a/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx +++ b/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx @@ -13,6 +13,7 @@ const query: ElasticsearchQuery = { query: '', metrics: [{ id: '1', type: 'count' }], bucketAggs: [{ type: 'date_histogram', id: '2' }], + filters: [] }; describe('ElasticsearchQueryContext', () => { diff --git a/src/components/QueryEditor/ElasticsearchQueryContext.tsx b/src/components/QueryEditor/ElasticsearchQueryContext.tsx index 7595f75..6c8dddf 100644 --- a/src/components/QueryEditor/ElasticsearchQueryContext.tsx +++ b/src/components/QueryEditor/ElasticsearchQueryContext.tsx @@ -8,6 +8,7 @@ import { ElasticsearchQuery } from '@/types'; import { createReducer as createBucketAggsReducer } from './BucketAggregationsEditor/state/reducer'; import { reducer as metricsReducer } from './MetricAggregationsEditor/state/reducer'; +import { reducer as filtersReducer } from './FilterEditor/state/reducer'; import { aliasPatternReducer, queryReducer, initQuery, initExploreQuery } from './state'; import { getHook } from '@/utils/context'; import { Provider, useDispatch } from "react-redux"; @@ -64,10 +65,11 @@ export const ElasticsearchProvider = withStore(({ [onChange] ); - const reducer = combineReducers>({ + const reducer = combineReducers>({ query: queryReducer, alias: aliasPatternReducer, metrics: metricsReducer, + filters: filtersReducer, bucketAggs: createBucketAggsReducer(datasource.timeField), }); @@ -78,7 +80,7 @@ export const ElasticsearchProvider = withStore(({ reducer ); - const isUninitialized = !query.metrics || !query.bucketAggs || query.query === undefined; + const isUninitialized = !query.metrics || !query.filters || !query.bucketAggs || query.query === undefined; const [shouldRunInit, setShouldRunInit] = useState(isUninitialized); diff --git a/src/components/QueryEditor/FilterEditor/index.tsx b/src/components/QueryEditor/FilterEditor/index.tsx new file mode 100644 index 0000000..e30bdbc --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/index.tsx @@ -0,0 +1,149 @@ +import React, { useRef } from 'react'; + +import { useDispatch } from '@/hooks/useStatelessReducer'; +import { IconButton } from '../../IconButton'; +import { useQuery } from '../ElasticsearchQueryContext'; +import { QueryEditorRow } from '../QueryEditorRow'; + +import { QueryFilter } from '@/types'; +import { InlineSegmentGroup, Input, Segment, SegmentAsync, Tooltip } from '@grafana/ui'; +import { + addFilter, + removeFilter, + toggleFilterVisibility, + changeFilterField, + changeFilterOperation, + changeFilterValue, +} from '@/components/QueryEditor/FilterEditor/state/actions'; +import { segmentStyles } from '@/components/QueryEditor/styles'; +import { useFields } from '@/hooks/useFields'; +import { newFilterId } from '@/utils/uid'; +import { filterOperations } from '@/queryDef'; +import { hasWhiteSpace, isSet } from '@/utils'; + +interface FilterEditorProps { + onSubmit: () => void; +} + +function filterErrors(filter: QueryFilter): string[] { + const errors: string[] = []; + + if (!isSet(filter.filter.key)) { + errors.push('Field is not set'); + } + + if (!isSet(filter.filter.operator)) { + errors.push('Operator is not set'); + } + + if (!['exists', 'not exists'].includes(filter.filter.operator) && !isSet(filter.filter.value)) { + errors.push('Value is not set'); + } + + if (['term', 'not term'].includes(filter.filter.operator) && filter.filter.value && hasWhiteSpace(filter.filter.value)) { + errors.push('Term cannot have whitespace in value'); + } + + return errors; +} + +export const FilterEditor = ({ onSubmit }: FilterEditorProps) => { + const dispatch = useDispatch(); + const { filters } = useQuery(); + + return ( + <> + {filters?.map((filter, index) => { + const errors = filterErrors(filter) + return ( + 0 ? ( + + Filter + + ): 'Filter'} + hidden={filter.hide} + onHideClick={() => { + dispatch(toggleFilterVisibility(filter.id)); + onSubmit(); + }} + onRemoveClick={() => { + dispatch(removeFilter(filter.id)); + onSubmit(); + }} + > + + + {index === 0 && dispatch(addFilter(newFilterId()))} + />} + + ) + })} + + ); +}; + +interface FilterEditorRowProps { + value: QueryFilter; + onSubmit: () => void; +} + +export const FilterEditorRow = ({ value, onSubmit }: FilterEditorRowProps) => { + const dispatch = useDispatch(); + const getFields = useFields('filters', 'startsWith'); + const valueInputRef = useRef(null); + + return ( + <> + + { + dispatch(changeFilterField({ id: value.id, field: e.value ?? '' })); + if (['exists', 'not exists'].includes(value.filter.operator) || isSet(value.filter.value)) { + onSubmit(); + } + // Auto focus the value input when a field is selected + setTimeout(() => valueInputRef.current?.focus(), 100); + }} + placeholder="Select Field" + value={value.filter.key} + /> +
+ op.value === value.filter.operator)} + options={filterOperations} + onChange={(e) => { + let op = e.value ?? filterOperations[0].value; + dispatch(changeFilterOperation({ id: value.id, op: op })); + if (['exists', 'not exists'].includes(op) || isSet(value.filter.value)) { + onSubmit(); + } + }} + /> +
+ {!['exists', 'not exists'].includes(value.filter.operator) && ( + dispatch(changeFilterValue({ id: value.id, value: e.currentTarget.value }))} + onKeyUp={(e) => { + if (e.key === 'Enter') { + onSubmit(); + } + }} + /> + )} + + + ); +}; diff --git a/src/components/QueryEditor/FilterEditor/state/actions.ts b/src/components/QueryEditor/FilterEditor/state/actions.ts new file mode 100644 index 0000000..ae9b7b5 --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/state/actions.ts @@ -0,0 +1,10 @@ +import { createAction } from '@reduxjs/toolkit'; + +import { QueryFilter } from '@/types'; + +export const addFilter = createAction('@filters/add'); +export const removeFilter = createAction('@filters/remove'); +export const toggleFilterVisibility = createAction('@filters/toggle_visibility'); +export const changeFilterField = createAction<{ id: QueryFilter['id']; field: string }>('@filters/change_field'); +export const changeFilterValue = createAction<{ id: QueryFilter['id']; value: string }>('@filters/change_value'); +export const changeFilterOperation = createAction<{ id: QueryFilter['id']; op: string }>('@filters/change_operation'); diff --git a/src/components/QueryEditor/FilterEditor/state/reducer.ts b/src/components/QueryEditor/FilterEditor/state/reducer.ts new file mode 100644 index 0000000..ed8ed87 --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/state/reducer.ts @@ -0,0 +1,100 @@ +import { Action } from '@reduxjs/toolkit'; +import { defaultFilter } from '@/queryDef'; +import { ElasticsearchQuery } from '@/types'; +import { initExploreQuery, initQuery } from '../../state'; + +import { + addFilter, + changeFilterField, + changeFilterOperation, + changeFilterValue, + removeFilter, + toggleFilterVisibility, +} from './actions'; + +export const reducer = (state: ElasticsearchQuery['filters'], action: Action): ElasticsearchQuery['filters'] => { + // console.log('Running filters reducer with action:', action, state); + + if (addFilter.match(action)) { + return [...state!, defaultFilter(action.payload)]; + } + + if (removeFilter.match(action)) { + const filterToRemove = state!.find((m) => m.id === action.payload)!; + const resultingFilters = state!.filter((filter) => filterToRemove.id !== filter.id); + if (resultingFilters.length === 0) { + return [defaultFilter()]; + } + return resultingFilters; + } + + if (changeFilterField.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + key: action.payload.field, + } + }; + }); + } + + if (changeFilterOperation.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + operator: action.payload.op, + } + }; + }); + } + + if (changeFilterValue.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + value: action.payload.value, + } + }; + }); + } + + if (toggleFilterVisibility.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload) { + return filter; + } + + return { + ...filter, + hide: !filter.hide, + }; + }); + } + + if (initQuery.match(action) || initExploreQuery.match(action)) { + if (state && state.length > 0) { + return state; + } + return [defaultFilter()]; + } + + return state; +}; diff --git a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx index e174716..2c8575a 100644 --- a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx +++ b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx @@ -24,6 +24,7 @@ describe('Metric Editor', () => { query: '', metrics: [avg], bucketAggs: [defaultBucketAgg('2')], + filters: [], }; const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); @@ -62,6 +63,7 @@ describe('Metric Editor', () => { query: '', metrics: [count], bucketAggs: [], + filters: [], }; const wrapper = ({ children }: PropsWithChildren<{}>) => ( diff --git a/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx index 5032c52..a0b26ad 100644 --- a/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx +++ b/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx @@ -27,6 +27,7 @@ describe('Settings Editor', () => { }, ], bucketAggs: [], + filters: [], }; const onChange = jest.fn(); @@ -102,6 +103,7 @@ describe('Settings Editor', () => { }, ], bucketAggs: [], + filters: [], }; const onChange = jest.fn(); diff --git a/src/components/QueryEditor/index.test.tsx b/src/components/QueryEditor/index.test.tsx index f51cf03..c932a8a 100644 --- a/src/components/QueryEditor/index.test.tsx +++ b/src/components/QueryEditor/index.test.tsx @@ -20,6 +20,7 @@ describe('QueryEditor', () => { ], // Even if present, this shouldn't be shown in the UI bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; render(); @@ -38,6 +39,7 @@ describe('QueryEditor', () => { }, ], bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; render(); diff --git a/src/components/QueryEditor/index.tsx b/src/components/QueryEditor/index.tsx index 773c435..9059fee 100644 --- a/src/components/QueryEditor/index.tsx +++ b/src/components/QueryEditor/index.tsx @@ -25,6 +25,7 @@ import { QueryTypeSelector } from './QueryTypeSelector'; import { getHook } from '@/utils/context'; import { LuceneQueryEditor } from '@/components/LuceneQueryEditor'; import { useDatasourceFields } from '@/datasource/utils'; +import { FilterEditor } from '@/components/QueryEditor/FilterEditor'; export type ElasticQueryEditorProps = QueryEditorProps; @@ -133,6 +134,7 @@ const QueryEditorForm = ({ value, onRunQuery }: Props) => { value={value?.query} onSubmit={onSubmitCB}/> + {showBucketAggregationsEditor && } diff --git a/src/dataquery.gen.ts b/src/dataquery.gen.ts index bd5dc47..0b70b88 100644 --- a/src/dataquery.gen.ts +++ b/src/dataquery.gen.ts @@ -9,6 +9,7 @@ // Run 'make gen-cue' from repository root to regenerate. import { DataQuery } from '@grafana/schema'; +import { AdHocVariableFilter } from '@grafana/data'; export const DataQueryModelVersion = Object.freeze([0, 0]); @@ -128,6 +129,12 @@ export interface BaseMetricAggregation { type: MetricAggregationType; } +export interface QueryFilter { + hide?: boolean; + id: string; + filter: AdHocVariableFilter; +} + export interface PipelineVariable { name: string; pipelineAgg: string; @@ -392,6 +399,10 @@ export interface Elasticsearch extends DataQuery { * List of metric aggregations */ metrics?: MetricAggregation[]; + /** + * List of filters + */ + filters?: QueryFilter[]; /** * Lucene query */ @@ -405,4 +416,5 @@ export interface Elasticsearch extends DataQuery { export const defaultElasticsearch: Partial = { bucketAggs: [], metrics: [], + filters: [], }; diff --git a/src/datasource/base.ts b/src/datasource/base.ts index e90f47d..2f3d624 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -16,9 +16,9 @@ import { TimeRange, } from '@grafana/data'; import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery, FieldCapabilitiesResponse } from '@/types'; -import { - DataSourceWithBackend, - getTemplateSrv, +import { + DataSourceWithBackend, + getTemplateSrv, TemplateSrv } from '@grafana/runtime'; import { QuickwitOptions } from 'quickwit'; import { getDataQuery } from 'QueryBuilder/elastic'; @@ -28,15 +28,15 @@ import { isMetricAggregationWithField } from 'components/QueryEditor/MetricAggre import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregationsEditor/utils'; import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations'; import ElasticsearchLanguageProvider from 'LanguageProvider'; -import { fieldTypeMap } from 'utils'; +import { fieldTypeMap, hasWhiteSpace } from 'utils'; import { addAddHocFilter } from 'modifyQuery'; import { getQueryResponseProcessor } from 'datasource/processResponse'; import { SECOND } from 'utils/time'; import { GConstructor } from 'utils/mixins'; -import { LuceneQuery } from '@/utils/lucene'; -import { uidMaker } from "@/utils/uid" +import { newFilterId, uidMaker } from '@/utils/uid'; import { DefaultsConfigOverrides } from 'store/defaults/conf'; +import { isSet } from '@/utils'; export type BaseQuickwitDataSourceConstructor = GConstructor @@ -122,18 +122,39 @@ export class BaseQuickwitDataSource return query; } - let lquery = LuceneQuery.parse(query.query ?? '') - switch (action.type) { - case 'ADD_FILTER': { - lquery = lquery.addFilter(action.options.key, action.options.value) - break; - } - case 'ADD_FILTER_OUT': { - lquery = lquery.addFilter(action.options.key, action.options.value, '-') - break; + const operationsMap: Record = { + 'ADD_FILTER': '=', + 'ADD_FILTER_OUT': '!=', + }; + const operation = operationsMap[action.type]; + + if (operation) { + // If the user has not added any filter, we can simply modify the last one (which is empty) + const len = query.filters?.length ?? 0; + if (len > 0) { + const last = query.filters![len - 1]; + if (!isSet(last.filter.key) && !isSet(last.filter.value)) { + last.filter.key = action.options.key; + last.filter.operator = operation; + last.filter.value = action.options.value; + return query; + } } + + query.filters?.push({ + id: newFilterId(), + hide: false, + filter: { + key: action.options.key, + operator: operation, + value: action.options.value, + }, + }); + } else { + console.warn('unsupported operation', action.type); } - return { ...query, query: lquery.toString() }; + + return { ...query }; } getDataQueryRequest(queryDef: TermsQuery, range: TimeRange, requestId?: string) { @@ -198,7 +219,7 @@ export class BaseQuickwitDataSource .map(field_capability => { return { text: field_capability.field_name, - type: fieldTypeMap[field_capability.type], + type: fieldTypeMap[field_capability.type], } }); const uniquefieldCapabilities = fieldCapabilities.filter((field_capability, index, self) => @@ -336,14 +357,36 @@ export class BaseQuickwitDataSource return bucketAgg; }; + const renderedQuery = (() => { + let q = this.interpolateLuceneQuery(query.query || '', scopedVars); + const queryFilters = query.filters + ?.filter((f) => { + if (f.hide) { + return false; + } + const hasValidValue = ( + ['exists', 'not exists'].includes(f.filter.operator) || isSet(f.filter.value) + ) && ( + !['term', 'not term'].includes(f.filter.operator) || !hasWhiteSpace(f.filter.value) + ) + + return isSet(f.filter.key) && hasValidValue && isSet(f.filter.operator) + }) + .map((f) => f.filter); + q = this.addAdHocFilters(q, queryFilters) + q = this.addAdHocFilters(q, filters) + return q + })() + const expandedQuery = { ...query, datasource: this.getRef(), - query: this.addAdHocFilters(this.interpolateLuceneQuery(query.query || '', scopedVars), filters), + query: renderedQuery, bucketAggs: query.bucketAggs?.map(interpolateBucketAgg), }; const finalQuery = JSON.parse(this.templateSrv.replace(JSON.stringify(expandedQuery), scopedVars)); + console.log('Final query', finalQuery.query) return finalQuery; } diff --git a/src/datasource/supplementaryQueries.ts b/src/datasource/supplementaryQueries.ts index 582d069..bf4f63d 100644 --- a/src/datasource/supplementaryQueries.ts +++ b/src/datasource/supplementaryQueries.ts @@ -102,6 +102,7 @@ export function withSupplementaryQueries { query: '', metrics: [defaultMetricAgg()], bucketAggs: [defaultBucketAgg()], + filters: [] }; const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); diff --git a/src/hooks/useFields.ts b/src/hooks/useFields.ts index 400e071..b4f2192 100644 --- a/src/hooks/useFields.ts +++ b/src/hooks/useFields.ts @@ -46,14 +46,17 @@ const toSelectableValue = ({ text }: MetricFindValue): SelectableValue = value: text, }); +type MatchType = 'contains' | 'startsWith' + /** * Returns a function to query the configured datasource for autocomplete values for the specified aggregation type or data types. * Each aggregation can be run on different types, for example avg only operates on numeric fields, geohash_grid only on geo_point fields. * If an aggregation type is provided, the promise will resolve with all fields suitable to be used as a field for the given aggregation. * If an array of types is providem the promise will resolve with all the fields matching the provided types. - * @param aggregationType the type of aggregation to get fields for + * @param type the type of aggregation to get fields for + * @param matchType the type of matching to use when filtering fields based on the query string. Defaults to 'contains'. */ -export const useFields = (type: AggregationType | string[]) => { +export const useFields = (type: AggregationType | string[], matchType: MatchType = 'contains') => { const datasource = useDatasource(); const range = useRange(); const filter = Array.isArray(type) ? type : getFilter(type); @@ -65,6 +68,13 @@ export const useFields = (type: AggregationType | string[]) => { rawFields = await lastValueFrom(datasource.getFields({aggregatable:true, type:filter, range:range})); } - return rawFields.filter(({ text }) => q === undefined || text.includes(q)).map(toSelectableValue); + return rawFields + .filter(({ text }) => { + if (q === undefined) { + return true; + } + return matchType === 'contains' ? text.includes(q) : text.startsWith(q) + }) + .map(toSelectableValue); }; }; diff --git a/src/hooks/useNextId.test.tsx b/src/hooks/useNextId.test.tsx index 1b0d287..0e76dcb 100644 --- a/src/hooks/useNextId.test.tsx +++ b/src/hooks/useNextId.test.tsx @@ -16,6 +16,7 @@ describe('useNextId', () => { query: '', metrics: [{ id: '1', type: 'avg' }], bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; const wrapper = ({ children }: PropsWithChildren<{}>) => { return ( diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index 8bd58b4..5b79c9e 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -5,7 +5,8 @@ import { AdHocVariableFilter } from '@grafana/data'; * Adds a label:"value" expression to the query. */ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): string { - if (!filter.key || !filter.value) { + const hasValidValue = ['exists', 'not exists'].includes(filter.operator) || !!filter.value + if (!filter.key || !hasValidValue) { return query; } @@ -39,6 +40,18 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str case '<': addHocFilter = `${key}:<${value}`; break; + case 'term': + addHocFilter = `${key}:${value}`; + break; + case 'not term': + addHocFilter = `-${key}:${value}`; + break; + case 'exists': + addHocFilter = `${key}:*`; + break; + case 'not exists': + addHocFilter = `-${key}:*`; + break; } return concatenate(query, addHocFilter,'AND'); } diff --git a/src/queryDef.ts b/src/queryDef.ts index 9fa54bb..7655f6e 100644 --- a/src/queryDef.ts +++ b/src/queryDef.ts @@ -6,7 +6,9 @@ import { MovingAverageModelOption, MetricAggregationType, DateHistogram, + QueryFilter, } from './types'; +import { newFilterId } from '@/utils/uid'; export const extendedStats: ExtendedStat[] = [ { label: 'Avg', value: 'avg' }, @@ -35,6 +37,19 @@ export function defaultBucketAgg(id = '1'): DateHistogram { return { type: 'date_histogram', id, settings: { interval: 'auto' } }; } +export const filterOperations = [ + { label: 'phrase', value: '=' }, + { label: 'not phrase', value: '!=' }, + { label: 'term', value: 'term' }, + { label: 'not term', value: 'not term' }, + { label: 'exists', value: 'exists' }, + { label: 'not exists', value: 'not exists' }, +]; + +export function defaultFilter(id = newFilterId()): QueryFilter { + return { id, filter: { key: '', operator: filterOperations[0].value, value: '' } }; +} + export const findMetricById = (metrics: MetricAggregation[], id: MetricAggregation['id']) => metrics.find((metric) => metric.id === id); diff --git a/src/utils/index.ts b/src/utils/index.ts index 8e5a826..518cf43 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -131,3 +131,8 @@ export const fieldTypeMap: Record = { float: 'number', scaled_float: 'number' }; + +export const isSet = (v: string) => v !== '' && v !== undefined && v !== null; + +export const hasWhiteSpace = (s: string) => /\s/g.test(s); + diff --git a/src/utils/uid.ts b/src/utils/uid.ts index bf51aac..9f029c7 100644 --- a/src/utils/uid.ts +++ b/src/utils/uid.ts @@ -1,3 +1,5 @@ +import { QueryFilter } from '@/dataquery.gen'; + export function uidMaker(prefix: string){ let i = 1; return { @@ -10,3 +12,7 @@ export function uidMaker(prefix: string){ } } } + +export function newFilterId(): QueryFilter['id'] { + return Math.floor(Math.random() * 100_000_000).toString() +} From 4407b1db59ad33ecb8d47e9138b59d6e6be4c7f0 Mon Sep 17 00:00:00 2001 From: Patrik Cyvoct Date: Tue, 21 Apr 2026 01:23:47 +0200 Subject: [PATCH 03/14] improve quick filter UX Signed-off-by: Patrik Cyvoct --- .../QueryEditor/FilterEditor/index.tsx | 71 +++++-- .../QueryEditor/FilterEditor/state/reducer.ts | 1 - src/datasource/base.ts | 173 +++++++++++++++--- src/queryDef.ts | 46 ++++- 4 files changed, 237 insertions(+), 54 deletions(-) diff --git a/src/components/QueryEditor/FilterEditor/index.tsx b/src/components/QueryEditor/FilterEditor/index.tsx index e30bdbc..6cd0db6 100644 --- a/src/components/QueryEditor/FilterEditor/index.tsx +++ b/src/components/QueryEditor/FilterEditor/index.tsx @@ -1,12 +1,13 @@ -import React, { useRef } from 'react'; +import React from 'react'; import { useDispatch } from '@/hooks/useStatelessReducer'; import { IconButton } from '../../IconButton'; -import { useQuery } from '../ElasticsearchQueryContext'; +import { useDatasource, useQuery, useRange } from '../ElasticsearchQueryContext'; import { QueryEditorRow } from '../QueryEditorRow'; import { QueryFilter } from '@/types'; -import { InlineSegmentGroup, Input, Segment, SegmentAsync, Tooltip } from '@grafana/ui'; +import { Icon, InlineSegmentGroup, Segment, SegmentAsync, Tooltip } from '@grafana/ui'; +import { MetricFindValue, SelectableValue } from '@grafana/data'; import { addFilter, removeFilter, @@ -18,7 +19,7 @@ import { import { segmentStyles } from '@/components/QueryEditor/styles'; import { useFields } from '@/hooks/useFields'; import { newFilterId } from '@/utils/uid'; -import { filterOperations } from '@/queryDef'; +import { categorizeFieldType, filterOperations, filterOperationsFor } from '@/queryDef'; import { hasWhiteSpace, isSet } from '@/utils'; interface FilterEditorProps { @@ -95,8 +96,24 @@ interface FilterEditorRowProps { export const FilterEditorRow = ({ value, onSubmit }: FilterEditorRowProps) => { const dispatch = useDispatch(); + const datasource = useDatasource(); + const range = useRange(); const getFields = useFields('filters', 'startsWith'); - const valueInputRef = useRef(null); + + const fieldCategory = categorizeFieldType(datasource.getFieldType?.(value.filter.key)); + const visibleOperations = filterOperationsFor(fieldCategory); + + const loadValues = async (query?: string): Promise>> => { + if (!isSet(value.filter.key) || !datasource.getTagValues) { + return []; + } + const values: MetricFindValue[] = await datasource.getTagValues({ key: value.filter.key, timeRange: range }); + const q = query?.toLowerCase(); + return values + .map((v) => String(v.text)) + .filter((text) => !q || text.toLowerCase().includes(q)) + .map((text) => ({ label: text, value: text })); + }; return ( <> @@ -107,20 +124,26 @@ export const FilterEditorRow = ({ value, onSubmit }: FilterEditorRowProps) => { loadOptions={getFields} reloadOptionsOnChange={true} onChange={(e) => { - dispatch(changeFilterField({ id: value.id, field: e.value ?? '' })); + const newKey = e.value ?? ''; + dispatch(changeFilterField({ id: value.id, field: newKey })); + // If the currently selected operator isn't valid for the new field, + // reset it to the first valid one so the UI stays consistent. + const newCategory = categorizeFieldType(datasource.getFieldType?.(newKey)); + const allowed = filterOperationsFor(newCategory); + if (!allowed.some((op) => op.value === value.filter.operator)) { + dispatch(changeFilterOperation({ id: value.id, op: allowed[0].value })); + } if (['exists', 'not exists'].includes(value.filter.operator) || isSet(value.filter.value)) { onSubmit(); } - // Auto focus the value input when a field is selected - setTimeout(() => valueInputRef.current?.focus(), 100); }} placeholder="Select Field" value={value.filter.key} /> -
+
op.value === value.filter.operator)} - options={filterOperations} + options={visibleOperations} onChange={(e) => { let op = e.value ?? filterOperations[0].value; dispatch(changeFilterOperation({ id: value.id, op: op })); @@ -129,17 +152,31 @@ export const FilterEditorRow = ({ value, onSubmit }: FilterEditorRowProps) => { } }} /> + +
is / is not — phrase match. On a keyword field this is exact equality; on a text field it matches a contiguous sequence of tokens anywhere in the value.
+
contains / does not contain — single-token match (no whitespace allowed). On a keyword field this is exact equality; on a text field it matches any document whose analyzed tokens include this term. Not a substring match — “germ” will not match “germany”.
+
> / < — numeric or date range. Shown only for numeric and date fields.
+
exists / does not exist — presence check, no value needed.
+
+ } + placement="top" + > + +
{!['exists', 'not exists'].includes(value.filter.operator) && ( - dispatch(changeFilterValue({ id: value.id, value: e.currentTarget.value }))} - onKeyUp={(e) => { - if (e.key === 'Enter') { - onSubmit(); - } + onChange={(e) => { + dispatch(changeFilterValue({ id: value.id, value: e.value ?? '' })); + onSubmit(); }} /> )} diff --git a/src/components/QueryEditor/FilterEditor/state/reducer.ts b/src/components/QueryEditor/FilterEditor/state/reducer.ts index ed8ed87..4ebc3b5 100644 --- a/src/components/QueryEditor/FilterEditor/state/reducer.ts +++ b/src/components/QueryEditor/FilterEditor/state/reducer.ts @@ -13,7 +13,6 @@ import { } from './actions'; export const reducer = (state: ElasticsearchQuery['filters'], action: Action): ElasticsearchQuery['filters'] => { - // console.log('Running filters reducer with action:', action, state); if (addFilter.match(action)) { return [...state!, defaultFilter(action.payload)]; diff --git a/src/datasource/base.ts b/src/datasource/base.ts index 2f3d624..d7b132c 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -3,6 +3,7 @@ import { Observable, lastValueFrom, from, of, map, mergeMap } from 'rxjs'; import { AbstractQuery, AdHocVariableFilter, + AppEvents, CoreApp, DataFrame, DataQueryRequest, @@ -14,10 +15,12 @@ import { QueryFixAction, ScopedVars, TimeRange, + ToggleFilterAction, } from '@grafana/data'; import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery, FieldCapabilitiesResponse } from '@/types'; import { DataSourceWithBackend, + getAppEvents, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; import { QuickwitOptions } from 'quickwit'; @@ -63,6 +66,18 @@ export class BaseQuickwitDataSource defaults?: DefaultsConfigOverrides }; languageProvider: ElasticsearchLanguageProvider; + // Populated lazily by getFields(). Used by modifyQuery to pick an operator + // that's safe for the field's type (phrase queries require positions indexed, + // which Quickwit text fields don't have by default). + private fieldTypes: Record = {}; + // Tracks (key|value|operator) triples we've already warned about on this + // datasource instance so the same filter doesn't spam a toast on every + // dashboard re-render. + private warnedAdHocFilters: Set = new Set(); + + getFieldType(name: string): string | undefined { + return this.fieldTypes[name]; + } constructor( @@ -79,6 +94,12 @@ export class BaseQuickwitDataSource this.queryEditorConfig = settingsData.queryEditorConfig || {}; this.languageProvider = new ElasticsearchLanguageProvider(this); this.annotations = {}; + + // Warm the schema cache so modifyQuery can pick the right operator even when + // the query editor hasn't mounted (e.g. dashboard view mode). Fire-and-forget. + if (this.index) { + lastValueFrom(this.getFields()).catch(() => {}); + } } query(request: DataQueryRequest): Observable { @@ -121,39 +142,85 @@ export class BaseQuickwitDataSource if (!action.options) { return query; } + if (action.type !== 'ADD_FILTER' && action.type !== 'ADD_FILTER_OUT') { + console.warn('unsupported operation', action.type); + return query; + } - const operationsMap: Record = { - 'ADD_FILTER': '=', - 'ADD_FILTER_OUT': '!=', - }; - const operation = operationsMap[action.type]; - - if (operation) { - // If the user has not added any filter, we can simply modify the last one (which is empty) - const len = query.filters?.length ?? 0; - if (len > 0) { - const last = query.filters![len - 1]; - if (!isSet(last.filter.key) && !isSet(last.filter.value)) { - last.filter.key = action.options.key; - last.filter.operator = operation; - last.filter.value = action.options.value; - return query; - } - } + const key = action.options.key; + const rawValue = String(action.options.value ?? ''); + const negate = action.type === 'ADD_FILTER_OUT'; + return this.addFilterToQuery(query, key, rawValue, negate); + } - query.filters?.push({ - id: newFilterId(), - hide: false, - filter: { - key: action.options.key, - operator: operation, - value: action.options.value, - }, + /** + * Newer Grafana interface (DataSourceWithToggleableQueryFiltersSupport) used + * by the Logs panel in dashboards. Preferred over modifyQuery when present. + */ + toggleQueryFilter(query: ElasticsearchQuery, filter: ToggleFilterAction): ElasticsearchQuery { + const key = filter.options.key; + const rawValue = String(filter.options.value ?? ''); + const negate = filter.type === 'FILTER_OUT'; + + // If the same (key, value) filter is already present, toggle it off. + const existingIdx = (query.filters ?? []).findIndex( + (f) => f.filter.key === key && f.filter.value === rawValue + ); + if (existingIdx !== -1) { + const next = [...(query.filters ?? [])]; + next.splice(existingIdx, 1); + return { ...query, filters: next.length ? next : [{ id: newFilterId(), filter: { key: '', operator: '=', value: '' } }] }; + } + return this.addFilterToQuery(query, key, rawValue, negate); + } + + queryHasFilter(query: ElasticsearchQuery, filter: { key: string; value: string }): boolean { + return (query.filters ?? []).some( + (f) => f.filter.key === filter.key && f.filter.value === filter.value && !f.hide + ); + } + + private addFilterToQuery(query: ElasticsearchQuery, key: string, rawValue: string, negate: boolean): ElasticsearchQuery { + const fieldType = this.fieldTypes[key]; + const isText = fieldType === 'text'; + + // Text field + value with whitespace: no single filter can represent this. + // Quickwit text fields lack positions so phrase fails, and `term` can't + // express whitespace. Warn the user and abort rather than add a broken or + // approximate filter. + if (isText && hasWhiteSpace(rawValue)) { + getAppEvents().publish({ + type: AppEvents.alertWarning.name, + payload: [ + `Cannot filter on "${key}"`, + `Quickwit text fields don't index positions by default, so "${rawValue}" can't be filtered as a phrase. Add this filter manually or edit the Lucene query.`, + ], }); - } else { - console.warn('unsupported operation', action.type); + return query; } + const value = rawValue; + const operator = isText && value !== '' + ? (negate ? 'not term' : 'term') + : (negate ? '!=' : '='); + + // If the user hasn't populated any filter yet, reuse the trailing empty one. + const len = query.filters?.length ?? 0; + if (len > 0) { + const last = query.filters![len - 1]; + if (!isSet(last.filter.key) && !isSet(last.filter.value)) { + last.filter.key = key; + last.filter.operator = operator; + last.filter.value = value; + return query; + } + } + query.filters?.push({ + id: newFilterId(), + hide: false, + filter: { key, operator, value }, + }); + return { ...query }; } @@ -195,6 +262,18 @@ export class BaseQuickwitDataSource end_timestamp: Math.ceil(range.to.valueOf()/SECOND), })).pipe( map((field_capabilities_response: FieldCapabilitiesResponse) => { + // Cache field → type on the datasource for modifyQuery to consult. + // Quickwit routes phrase queries to the text variant first on multi-indexed + // fields (text+keyword), and text fields don't index positions by default. + // So prefer 'text' when present — it drives safer operator choices downstream. + for (const [name, caps] of Object.entries(field_capabilities_response.fields)) { + const typeKeys = Object.keys(caps); + const chosen = typeKeys.includes('text') ? 'text' : typeKeys[0]; + if (chosen) { + this.fieldTypes[name] = chosen; + } + } + const shouldAddField = (field: any) => { if (spec.aggregatable !== undefined && field.aggregatable !== spec.aggregatable) { return false @@ -386,7 +465,6 @@ export class BaseQuickwitDataSource }; const finalQuery = JSON.parse(this.templateSrv.replace(JSON.stringify(expandedQuery), scopedVars)); - console.log('Final query', finalQuery.query) return finalQuery; } @@ -396,7 +474,42 @@ export class BaseQuickwitDataSource } let finalQuery = query; adhocFilters.forEach((filter) => { - finalQuery = addAddHocFilter(finalQuery, filter); + const fieldType = this.fieldTypes[filter.key]; + const isText = fieldType === 'text'; + const value = filter.value ?? ''; + + // Text field + whitespace value + phrase operator would emit a phrase query, + // which fails on Quickwit text fields without positions indexed. Skip the + // filter — better a no-op than a broken panel. Fire a toast once per + // (key|value|operator) so the user knows why it's not applying, without + // spamming on every dashboard render. + if (isText && hasWhiteSpace(value) && (filter.operator === '=' || filter.operator === '!=')) { + const dedupKey = `${filter.key}|${value}|${filter.operator}`; + if (!this.warnedAdHocFilters.has(dedupKey)) { + this.warnedAdHocFilters.add(dedupKey); + getAppEvents().publish({ + type: AppEvents.alertWarning.name, + payload: [ + `Filter on "${filter.key}" was skipped`, + `Quickwit text fields don't index positions by default, so "${value}" can't be filtered as a phrase. Edit the Lucene query directly for this case.`, + ], + }); + } + return; + } + + // For text fields with single-token values, upgrade '=' to 'term' (and + // '!=' to 'not term') so the emitted query doesn't require positions. + let effective = filter; + if (isText && !hasWhiteSpace(value)) { + if (filter.operator === '=') { + effective = { ...filter, operator: 'term' }; + } else if (filter.operator === '!=') { + effective = { ...filter, operator: 'not term' }; + } + } + + finalQuery = addAddHocFilter(finalQuery, effective); }); return finalQuery; diff --git a/src/queryDef.ts b/src/queryDef.ts index 7655f6e..0b7cc0e 100644 --- a/src/queryDef.ts +++ b/src/queryDef.ts @@ -37,15 +37,49 @@ export function defaultBucketAgg(id = '1'): DateHistogram { return { type: 'date_histogram', id, settings: { interval: 'auto' } }; } -export const filterOperations = [ - { label: 'phrase', value: '=' }, - { label: 'not phrase', value: '!=' }, - { label: 'term', value: 'term' }, - { label: 'not term', value: 'not term' }, +export type FilterFieldCategory = 'text' | 'keyword' | 'number' | 'date' | 'boolean' | 'other'; + +export interface FilterOperation { + label: string; + value: string; + // Field categories this operation applies to. Undefined = always available. + types?: FilterFieldCategory[]; +} + +export const filterOperations: FilterOperation[] = [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' }, + { label: 'contains', value: 'term', types: ['text'] }, + { label: 'does not contain', value: 'not term', types: ['text'] }, + { label: '>', value: '>', types: ['number', 'date'] }, + { label: '<', value: '<', types: ['number', 'date'] }, { label: 'exists', value: 'exists' }, - { label: 'not exists', value: 'not exists' }, + { label: 'does not exist', value: 'not exists' }, ]; +/** + * Map a Quickwit/Elasticsearch raw field type (as returned by _field_caps) to + * a coarser category used to decide which operators to expose in the filter UI. + */ +export function categorizeFieldType(rawType?: string): FilterFieldCategory { + if (!rawType) {return 'other';} + if (rawType === 'text') {return 'text';} + if (rawType === 'keyword') {return 'keyword';} + if (['long', 'integer', 'int', 'short', 'byte', 'double', 'float', 'scaled_float', 'half_float', 'unsigned_long'].includes(rawType)) {return 'number';} + if (['date', 'date_nanos'].includes(rawType)) {return 'date';} + if (rawType === 'boolean' || rawType === 'bool') {return 'boolean';} + return 'other'; +} + +/** + * Operations available for a given field category. When the category is 'other' + * (unknown field) we return everything so the user still has full control. + */ +export function filterOperationsFor(category: FilterFieldCategory): FilterOperation[] { + if (category === 'other') {return filterOperations;} + return filterOperations.filter((op) => !op.types || op.types.includes(category)); +} + export function defaultFilter(id = newFilterId()): QueryFilter { return { id, filter: { key: '', operator: filterOperations[0].value, value: '' } }; } From 0305bff19754fef767abf5ac5ab9d48ab8579b86 Mon Sep 17 00:00:00 2001 From: Patrik Cyvoct Date: Tue, 21 Apr 2026 13:17:41 +0200 Subject: [PATCH 04/14] test(e2e): handle Grafana 13 What's-New modal and datasource picker overlay Signed-off-by: Patrik Cyvoct --- tests/quickwit.spec.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/quickwit.spec.ts b/tests/quickwit.spec.ts index a2fa64d..f7c0fb7 100644 --- a/tests/quickwit.spec.ts +++ b/tests/quickwit.spec.ts @@ -1,4 +1,5 @@ import { test, expect } from '@grafana/plugin-e2e'; +import type { Page } from '@playwright/test'; // Grafana (inside Docker) reaches Quickwit via the docker service name. // The test process (on the host) reaches the same Quickwit via localhost port mapping. @@ -20,17 +21,22 @@ test.afterEach(async () => { test('create datasource and explore logs', async ({ page }) => { let datasourceUid: string; + const quickwitDatasourceButton = page.locator('button').filter({ hasText: /^Quickwit$/ }).first(); await test.step('look for the plugin in the list of datasources', async () => { await page.goto('/connections/datasources/new'); + await dismissWhatsNewModal(page); await page.getByPlaceholder('Filter by name or type').fill('quickwit'); - await expect(page.getByText('Quickwit', { exact: true })).toBeVisible(); + await expect(quickwitDatasourceButton).toBeVisible(); }); await test.step('create datasource via UI', async () => { await page.goto('/connections/datasources/new'); + await dismissWhatsNewModal(page); await page.getByPlaceholder('Filter by name or type').fill('quickwit'); - await page.getByText('Quickwit', { exact: true }).click(); + // Grafana 13 sometimes leaves a portal overlay mounted above the picker. + // Force the click once the correct button is resolved. + await quickwitDatasourceButton.click({ force: true }); // Wait for the config form to load await expect(page.getByText('Index settings')).toBeVisible(); @@ -106,3 +112,15 @@ async function ingestDummyLogs(runId: string) { throw new Error('Ingest failed: no searchable logs'); } } + +async function dismissWhatsNewModal(page: Page) { + const dialog = page.getByRole('dialog', { name: "What's new in Grafana" }); + const closeButton = dialog.getByRole('button', { name: 'Close' }); + + await closeButton.waitFor({ state: 'visible', timeout: 3000 }).catch(() => {}); + + if (await dialog.isVisible().catch(() => false)) { + await closeButton.click({ force: true }); + await expect(dialog).toBeHidden({ timeout: 10000 }); + } +} From 5da4a083e4983d16eecddf83b866eb03972ae9c1 Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Sat, 25 Apr 2026 17:06:19 -0400 Subject: [PATCH 05/14] Add tests capturing current processLogsDataFrame behavior Covers: configured logMessageField (single and multiple fields), missing field handling, empty dataframes, log-volume skip, no-refId skip, and the case where no logMessageField is configured (no synthetic field inserted). Refs quickwit-oss/quickwit-datasource#165 --- src/datasource/processResponse.test.ts | 131 +++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 src/datasource/processResponse.test.ts diff --git a/src/datasource/processResponse.test.ts b/src/datasource/processResponse.test.ts new file mode 100644 index 0000000..de63091 --- /dev/null +++ b/src/datasource/processResponse.test.ts @@ -0,0 +1,131 @@ +import { DataFrame, Field, FieldType } from '@grafana/data'; +import { processLogsDataFrame } from './processResponse'; + +function makeField(name: string, type: FieldType, values: any[]): Field { + return { name, type, config: {}, values }; +} + +function makeDataFrame(fields: Field[], refId = 'A'): DataFrame { + return { + refId, + fields, + length: fields[0]?.values.length ?? 0, + }; +} + +function makeDatasource(overrides: { logMessageField?: string; dataLinks?: any[] } = {}) { + return { + logMessageField: overrides.logMessageField ?? '', + dataLinks: overrides.dataLinks ?? [], + } as any; +} + +describe('processLogsDataFrame', () => { + describe('with logMessageField configured', () => { + it('inserts synthetic $qw_message field from configured field', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000, 2000]), + makeField('line', FieldType.string, ['hello world', 'goodbye world']), + makeField('level', FieldType.string, ['info', 'error']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[0].name).toBe('timestamp'); + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values).toEqual(['hello world', 'goodbye world']); + }); + + it('joins multiple configured fields with key=value format', () => { + const ds = makeDatasource({ logMessageField: 'method,path,status' }); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('method', FieldType.string, ['GET']), + makeField('path', FieldType.string, ['/blog']), + makeField('status', FieldType.string, ['200']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('method=GET path=/blog status=200'); + }); + + it('inserts empty $qw_message when configured field does not exist', () => { + const ds = makeDatasource({ logMessageField: 'nonexistent' }); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('line', FieldType.string, ['hello']), + ]); + + processLogsDataFrame(ds, df); + + // $qw_message is still inserted but with empty values + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe(''); + }); + }); + + describe('without logMessageField configured', () => { + it('does not insert any synthetic field', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('attributes.controller', FieldType.string, ['BlogController']), + makeField('attributes.method', FieldType.string, ['GET']), + ]); + + processLogsDataFrame(ds, df); + + // No $qw_message field — current behavior is to do nothing + const fieldNames = df.fields.map((f) => f.name); + expect(fieldNames).not.toContain('$qw_message'); + }); + }); + + describe('edge cases', () => { + it('skips empty dataframes', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df = makeDataFrame([]); + + processLogsDataFrame(ds, df); + + expect(df.fields.length).toBe(0); + }); + + it('skips log-volume dataframes', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df = makeDataFrame( + [ + makeField('timestamp', FieldType.time, [1000]), + makeField('line', FieldType.string, ['hello']), + ], + 'log-volume-A' + ); + + processLogsDataFrame(ds, df); + + // Should not have inserted $qw_message + const fieldNames = df.fields.map((f) => f.name); + expect(fieldNames).not.toContain('$qw_message'); + }); + + it('skips dataframes with no refId', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df: DataFrame = { + refId: undefined, + fields: [ + makeField('timestamp', FieldType.time, [1000]), + makeField('line', FieldType.string, ['hello']), + ], + length: 1, + }; + + processLogsDataFrame(ds, df); + + const fieldNames = df.fields.map((f) => f.name); + expect(fieldNames).not.toContain('$qw_message'); + }); + }); +}); From e71bd3927b51639cb3546a1598e2ba088b38ca8b Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Sat, 25 Apr 2026 17:17:06 -0400 Subject: [PATCH 06/14] Add fallback log message display for OTEL logs When logMessageField is unconfigured or its value is empty for a row, build a display line by trying well-known OTEL fields (body.message, attributes.message) then falling back to a key=value summary of document fields. Metadata fields (pod info, node labels, sort, etc.) are excluded from the summary and attributes. prefixes are stripped for readability. The fallback is per-row so mixed log types (structured logger output, lograge requests, raw nginx lines) each show the best available content. Refs quickwit-oss/quickwit-datasource#165 --- src/datasource/processResponse.test.ts | 126 ++++++++++++++++++++++--- src/datasource/processResponse.ts | 114 ++++++++++++++++------ 2 files changed, 201 insertions(+), 39 deletions(-) diff --git a/src/datasource/processResponse.test.ts b/src/datasource/processResponse.test.ts index de63091..3843b58 100644 --- a/src/datasource/processResponse.test.ts +++ b/src/datasource/processResponse.test.ts @@ -22,7 +22,7 @@ function makeDatasource(overrides: { logMessageField?: string; dataLinks?: any[] describe('processLogsDataFrame', () => { describe('with logMessageField configured', () => { - it('inserts synthetic $qw_message field from configured field', () => { + it('uses configured field value', () => { const ds = makeDatasource({ logMessageField: 'line' }); const df = makeDataFrame([ makeField('timestamp', FieldType.time, [1000, 2000]), @@ -32,7 +32,6 @@ describe('processLogsDataFrame', () => { processLogsDataFrame(ds, df); - expect(df.fields[0].name).toBe('timestamp'); expect(df.fields[1].name).toBe('$qw_message'); expect(df.fields[1].values).toEqual(['hello world', 'goodbye world']); }); @@ -52,35 +51,139 @@ describe('processLogsDataFrame', () => { expect(df.fields[1].values[0]).toBe('method=GET path=/blog status=200'); }); - it('inserts empty $qw_message when configured field does not exist', () => { + it('falls back when configured field does not exist', () => { const ds = makeDatasource({ logMessageField: 'nonexistent' }); const df = makeDataFrame([ makeField('timestamp', FieldType.time, [1000]), - makeField('line', FieldType.string, ['hello']), + makeField('body.message', FieldType.string, ['the real message']), ]); processLogsDataFrame(ds, df); - // $qw_message is still inserted but with empty values expect(df.fields[1].name).toBe('$qw_message'); - expect(df.fields[1].values[0]).toBe(''); + expect(df.fields[1].values[0]).toBe('the real message'); + }); + + it('falls back when configured field value is empty for a row', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000, 2000]), + makeField('line', FieldType.string, ['has content', '']), + makeField('body.message', FieldType.string, ['', 'fallback message']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('has content'); + expect(df.fields[1].values[1]).toBe('fallback message'); }); }); - describe('without logMessageField configured', () => { - it('does not insert any synthetic field', () => { + describe('OTEL fallback (no logMessageField)', () => { + it('picks body.message when present', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('body.message', FieldType.string, ['GET /assets/app.js HTTP/1.1 200']), + makeField('body.stream', FieldType.string, ['stdout']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('GET /assets/app.js HTTP/1.1 200'); + }); + + it('picks attributes.message when body.message is absent', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('attributes.message', FieldType.string, ['SSO user already exists']), + makeField('attributes.severity', FieldType.string, ['INFO']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('SSO user already exists'); + }); + + it('prefers body.message over attributes.message', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('body.message', FieldType.string, ['from body']), + makeField('attributes.message', FieldType.string, ['from attributes']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('from body'); + }); + + it('builds key=value summary when no well-known fields exist', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('attributes.method', FieldType.string, ['GET']), + makeField('attributes.path', FieldType.string, ['/blog']), + makeField('attributes.status', FieldType.number, [200]), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('method=GET path=/blog status=200'); + }); + + it('strips attributes. prefix in key=value summary', () => { const ds = makeDatasource(); const df = makeDataFrame([ makeField('timestamp', FieldType.time, [1000]), makeField('attributes.controller', FieldType.string, ['BlogController']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].values[0]).toBe('controller=BlogController'); + }); + + it('skips metadata fields in key=value summary', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), makeField('attributes.method', FieldType.string, ['GET']), + makeField('attributes.pod_name', FieldType.string, ['rx-production-abc123']), + makeField('attributes.node_labels.arch', FieldType.string, ['amd64']), + makeField('sort', FieldType.other, [[1684398201000]]), + makeField('severity_text', FieldType.string, ['INFO']), + makeField('body.stream', FieldType.string, ['stdout']), ]); processLogsDataFrame(ds, df); - // No $qw_message field — current behavior is to do nothing - const fieldNames = df.fields.map((f) => f.name); - expect(fieldNames).not.toContain('$qw_message'); + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('method=GET'); + }); + + it('handles mixed log types per row', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000, 2000, 3000]), + makeField('attributes.message', FieldType.string, ['SSO login', '', '']), + makeField('attributes.method', FieldType.string, ['', 'GET', '']), + makeField('attributes.path', FieldType.string, ['', '/blog', '']), + makeField('body.message', FieldType.string, ['', '', 'raw nginx log line']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('SSO login'); + expect(df.fields[1].values[1]).toBe('method=GET path=/blog'); + expect(df.fields[1].values[2]).toBe('raw nginx log line'); }); }); @@ -106,7 +209,6 @@ describe('processLogsDataFrame', () => { processLogsDataFrame(ds, df); - // Should not have inserted $qw_message const fieldNames = df.fields.map((f) => f.name); expect(fieldNames).not.toContain('$qw_message'); }); diff --git a/src/datasource/processResponse.ts b/src/datasource/processResponse.ts index 6743cbf..45e8fe8 100644 --- a/src/datasource/processResponse.ts +++ b/src/datasource/processResponse.ts @@ -17,6 +17,58 @@ export function getQueryResponseProcessor(datasource: BaseQuickwitDataSource, re }; } function getCustomFieldName(fieldname: string) { return `$qw_${fieldname}`; } + +const OTEL_MESSAGE_FIELDS = ['body.message', 'attributes.message']; + +const SKIP_FIELD_PREFIXES = [ + 'attributes.pod_', 'attributes.node_labels.', 'attributes.namespace_labels.', + 'attributes.container_image', 'attributes.pod_owner', +]; +const SKIP_FIELD_NAMES = new Set([ + 'sort', 'severity_text', 'body.stream', +]); + +function isMetadataField(name: string, timeField: string): boolean { + if (name === timeField || SKIP_FIELD_NAMES.has(name)) { + return true; + } + return SKIP_FIELD_PREFIXES.some((prefix) => name.startsWith(prefix)); +} + +function stripPrefix(name: string): string { + if (name.startsWith('attributes.')) { + return name.slice('attributes.'.length); + } + if (name.startsWith('body.')) { + return name.slice('body.'.length); + } + return name; +} + +function buildFallbackMessage(dataFrame: DataFrame, rowIdx: number, timeFieldName: string): string { + for (const candidate of OTEL_MESSAGE_FIELDS) { + const field = dataFrame.fields.find((f) => f.name === candidate); + if (field) { + const val = field.values[rowIdx]; + if (val != null && val !== '') { + return String(val); + } + } + } + + const parts: string[] = []; + for (const field of dataFrame.fields) { + if (isMetadataField(field.name, timeFieldName) || field.type === FieldType.time) { + continue; + } + const val = field.values[rowIdx]; + if (val != null && val !== '') { + parts.push(`${stripPrefix(field.name)}=${val}`); + } + } + return parts.join(' '); +} + export function processLogsDataFrame(datasource: BaseQuickwitDataSource, dataFrame: DataFrame) { // Ignore log volume dataframe, no need to add links or a displayed message field. if (!dataFrame.refId || dataFrame.refId.startsWith('log-volume')) { @@ -26,39 +78,47 @@ export function processLogsDataFrame(datasource: BaseQuickwitDataSource, dataFra if (dataFrame.length===0 || dataFrame.fields.length === 0) { return; } - if (datasource.logMessageField) { - const messageFields = datasource.logMessageField.split(','); - let field_idx_list = []; - for (const messageField of messageFields) { - const field_idx = dataFrame.fields.findIndex((field) => field.name === messageField); - if (field_idx !== -1) { - field_idx_list.push(field_idx); - } + + const configuredFields = datasource.logMessageField ? datasource.logMessageField.split(',') : []; + const field_idx_list: number[] = []; + for (const messageField of configuredFields) { + const field_idx = dataFrame.fields.findIndex((field) => field.name === messageField); + if (field_idx !== -1) { + field_idx_list.push(field_idx); } - const displayedMessages = Array(dataFrame.length); - for (let idx = 0; idx < dataFrame.length; idx++) { - let displayedMessage = ""; - // If we have only one field, we assume the field name is obvious for the user and we don't need to show it. - if (field_idx_list.length === 1) { - displayedMessage = `${dataFrame.fields[field_idx_list[0]].values[idx]}`; - } else { - for (const field_idx of field_idx_list) { - displayedMessage += ` ${dataFrame.fields[field_idx].name}=${dataFrame.fields[field_idx].values[idx]}`; - } + } + + const timeFieldName = dataFrame.fields.find((f) => f.type === FieldType.time)?.name ?? ''; + const displayedMessages = Array(dataFrame.length); + + for (let idx = 0; idx < dataFrame.length; idx++) { + let displayedMessage = ""; + + if (field_idx_list.length === 1) { + displayedMessage = `${dataFrame.fields[field_idx_list[0]].values[idx] ?? ''}`; + } else if (field_idx_list.length > 1) { + for (const field_idx of field_idx_list) { + displayedMessage += ` ${dataFrame.fields[field_idx].name}=${dataFrame.fields[field_idx].values[idx]}`; } - displayedMessages[idx] = displayedMessage.trim(); + displayedMessage = displayedMessage.trim(); } - const newField: Field = { - name: getCustomFieldName('message'), - type: FieldType.string, - config: {}, - values: displayedMessages, - }; - const [timestamp, ...rest] = dataFrame.fields; - dataFrame.fields = [timestamp, newField, ...rest]; + if (!displayedMessage) { + displayedMessage = buildFallbackMessage(dataFrame, idx, timeFieldName); + } + + displayedMessages[idx] = displayedMessage; } + const newField: Field = { + name: getCustomFieldName('message'), + type: FieldType.string, + config: {}, + values: displayedMessages, + }; + const [timestamp, ...rest] = dataFrame.fields; + dataFrame.fields = [timestamp, newField, ...rest]; + if (!datasource.dataLinks.length) { return; } From 6a45d878719ee362a3e01b13f2cda7d498547cdb Mon Sep 17 00:00:00 2001 From: Patrik Cyvoct Date: Sun, 26 Apr 2026 19:15:30 +0200 Subject: [PATCH 07/14] fix: quick filters QoL Generate quoted phrase filters for text values that contain whitespace or punctuation, matching manually-entered Quickwit queries such as service_name:"auth-api" and attributes.grpc_message:"...". Keep unquoted term filters only for simple token values and escape term values correctly. Improve filter autocomplete with fuzzy matching for keys and values, and pass previous complete filters into later value lookups so suggestions narrow progressively. Add tests for phrase rendering, punctuated values, term escaping, fuzzy matching, and prior-filter selection. Signed-off-by: Patrik Cyvoct --- .../QueryEditor/FilterEditor/index.test.tsx | 50 ++++++ .../QueryEditor/FilterEditor/index.tsx | 51 ++++-- src/datasource/base.test.ts | 153 +++++++++++++++++- src/datasource/base.ts | 58 ++----- src/hooks/useFields.ts | 12 +- src/modifyQuery.ts | 5 +- src/utils/index.test.ts | 36 ++++- src/utils/index.ts | 124 ++++++++++++++ 8 files changed, 422 insertions(+), 67 deletions(-) create mode 100644 src/components/QueryEditor/FilterEditor/index.test.tsx diff --git a/src/components/QueryEditor/FilterEditor/index.test.tsx b/src/components/QueryEditor/FilterEditor/index.test.tsx new file mode 100644 index 0000000..b648254 --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/index.test.tsx @@ -0,0 +1,50 @@ +import { QueryFilter } from '@/types'; + +import { getPreviousAdHocFilters } from './index'; + +describe('FilterEditor helpers', () => { + it('returns only complete filters before the current filter', () => { + const filters: QueryFilter[] = [ + { + id: 'first', + filter: { key: 'service', operator: '=', value: 'frontend' }, + }, + { + id: 'hidden', + hide: true, + filter: { key: 'cluster', operator: '=', value: 'prod' }, + }, + { + id: 'incomplete', + filter: { key: 'namespace', operator: '=', value: '' }, + }, + { + id: 'current', + filter: { key: 'attributes.grpc_message', operator: '=', value: '' }, + }, + { + id: 'later', + filter: { key: 'status', operator: '=', value: '500' }, + }, + ]; + + expect(getPreviousAdHocFilters(filters, 'current')).toEqual([ + { key: 'service', operator: '=', value: 'frontend' }, + ]); + }); + + it('does not include previous term filters with whitespace values', () => { + const filters: QueryFilter[] = [ + { + id: 'invalid-term', + filter: { key: 'message', operator: 'term', value: 'invalid token' }, + }, + { + id: 'current', + filter: { key: 'status', operator: '=', value: '' }, + }, + ]; + + expect(getPreviousAdHocFilters(filters, 'current')).toEqual([]); + }); +}); diff --git a/src/components/QueryEditor/FilterEditor/index.tsx b/src/components/QueryEditor/FilterEditor/index.tsx index 6cd0db6..0c879dc 100644 --- a/src/components/QueryEditor/FilterEditor/index.tsx +++ b/src/components/QueryEditor/FilterEditor/index.tsx @@ -7,7 +7,7 @@ import { QueryEditorRow } from '../QueryEditorRow'; import { QueryFilter } from '@/types'; import { Icon, InlineSegmentGroup, Segment, SegmentAsync, Tooltip } from '@grafana/ui'; -import { MetricFindValue, SelectableValue } from '@grafana/data'; +import { AdHocVariableFilter, MetricFindValue, SelectableValue } from '@grafana/data'; import { addFilter, removeFilter, @@ -17,10 +17,9 @@ import { changeFilterValue, } from '@/components/QueryEditor/FilterEditor/state/actions'; import { segmentStyles } from '@/components/QueryEditor/styles'; -import { useFields } from '@/hooks/useFields'; import { newFilterId } from '@/utils/uid'; import { categorizeFieldType, filterOperations, filterOperationsFor } from '@/queryDef'; -import { hasWhiteSpace, isSet } from '@/utils'; +import { fuzzySearchSort, hasWhiteSpace, isSet } from '@/utils'; interface FilterEditorProps { onSubmit: () => void; @@ -48,6 +47,30 @@ function filterErrors(filter: QueryFilter): string[] { return errors; } +function isFilterComplete(filter: QueryFilter): boolean { + return !filter.hide && filterErrors(filter).length === 0; +} + +export function getPreviousAdHocFilters(filters: QueryFilter[] | undefined, currentId: QueryFilter['id']): AdHocVariableFilter[] { + const currentIndex = filters?.findIndex((filter) => filter.id === currentId) ?? -1; + if (!filters || currentIndex <= 0) { + return []; + } + + return filters + .slice(0, currentIndex) + .filter(isFilterComplete) + .map((filter) => filter.filter); +} + +function toFuzzyOptions(values: MetricFindValue[], query?: string): Array> { + return fuzzySearchSort( + values.map((value) => String(value.text)), + (text) => text, + query + ).map((text) => ({ label: text, value: text })); +} + export const FilterEditor = ({ onSubmit }: FilterEditorProps) => { const dispatch = useDispatch(); const { filters } = useQuery(); @@ -98,21 +121,27 @@ export const FilterEditorRow = ({ value, onSubmit }: FilterEditorRowProps) => { const dispatch = useDispatch(); const datasource = useDatasource(); const range = useRange(); - const getFields = useFields('filters', 'startsWith'); + const { filters } = useQuery(); + const previousFilters = getPreviousAdHocFilters(filters, value.id); const fieldCategory = categorizeFieldType(datasource.getFieldType?.(value.filter.key)); const visibleOperations = filterOperationsFor(fieldCategory); + const loadFields = async (query?: string): Promise>> => { + const values = await datasource.getTagKeys({ filters: previousFilters, timeRange: range }); + return toFuzzyOptions(values as MetricFindValue[], query); + }; + const loadValues = async (query?: string): Promise>> => { if (!isSet(value.filter.key) || !datasource.getTagValues) { return []; } - const values: MetricFindValue[] = await datasource.getTagValues({ key: value.filter.key, timeRange: range }); - const q = query?.toLowerCase(); - return values - .map((v) => String(v.text)) - .filter((text) => !q || text.toLowerCase().includes(q)) - .map((text) => ({ label: text, value: text })); + const values: MetricFindValue[] = await datasource.getTagValues({ + key: value.filter.key, + filters: previousFilters, + timeRange: range, + }); + return toFuzzyOptions(values, query); }; return ( @@ -121,7 +150,7 @@ export const FilterEditorRow = ({ value, onSubmit }: FilterEditorRowProps) => { { const newKey = e.value ?? ''; diff --git a/src/datasource/base.test.ts b/src/datasource/base.test.ts index d6f268e..22f1d43 100644 --- a/src/datasource/base.test.ts +++ b/src/datasource/base.test.ts @@ -1,4 +1,9 @@ -import { formatQuery, luceneEscape } from './base'; +import { AdHocVariableFilter } from '@grafana/data'; +import { from } from 'rxjs'; + +import { addAddHocFilter } from '../modifyQuery'; +import { ElasticsearchQuery } from '../types'; +import { BaseQuickwitDataSource, formatQuery, luceneEscape } from './base'; describe('BaseQuickwitDataSource', () => { describe('luceneEscape', () => { @@ -171,4 +176,150 @@ describe('BaseQuickwitDataSource', () => { }); }); }); + + describe('quick filters', () => { + const addFilterToQuery = ( + fieldTypes: Record, + query: ElasticsearchQuery, + key: string, + value: string, + negate = false + ) => { + return (BaseQuickwitDataSource.prototype as any).addFilterToQuery.call( + { fieldTypes }, + query, + key, + value, + negate + ) as ElasticsearchQuery; + }; + + const renderAdHocFilters = ( + fieldTypes: Record, + filters: AdHocVariableFilter[] + ) => { + return (BaseQuickwitDataSource.prototype as any).addAdHocFilters.call( + { fieldTypes }, + '', + filters + ) as string; + }; + + it('adds text filters with whitespace as phrase filters', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; + + const updatedQuery = addFilterToQuery( + { 'attributes.grpc_message': 'text' }, + query, + 'attributes.grpc_message', + 'Error:[(0) invalid token, ]' + ); + + expect(updatedQuery.filters?.[0].filter).toEqual({ + key: 'attributes.grpc_message', + operator: '=', + value: 'Error:[(0) invalid token, ]', + }); + }); + + it('renders text phrase filters with quoted Quickwit syntax', () => { + const result = renderAdHocFilters( + { 'attributes.grpc_message': 'text' }, + [{ + key: 'attributes.grpc_message', + operator: '=', + value: 'Error:[(0) invalid token, ]', + }] + ); + + expect(result).toBe('attributes.grpc_message:"Error:[(0) invalid token, ]"'); + }); + + it('renders negative text phrase filters with quoted Quickwit syntax', () => { + const result = renderAdHocFilters( + { 'attributes.grpc_message': 'text' }, + [{ + key: 'attributes.grpc_message', + operator: '!=', + value: 'Error:[(0) invalid token, ]', + }] + ); + + expect(result).toBe('-attributes.grpc_message:"Error:[(0) invalid token, ]"'); + }); + + it('keeps simple-token text filters as term filters', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; + + const updatedQuery = addFilterToQuery( + { 'attributes.grpc_message': 'text' }, + query, + 'attributes.grpc_message', + 'unavailable' + ); + + expect(updatedQuery.filters?.[0].filter.operator).toBe('term'); + }); + + it('keeps punctuated text filters as phrase filters', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; + + const updatedQuery = addFilterToQuery( + { service_name: 'text' }, + query, + 'service_name', + 'auth-api' + ); + + expect(updatedQuery.filters?.[0].filter).toEqual({ + key: 'service_name', + operator: '=', + value: 'auth-api', + }); + }); + + it('renders punctuated text filters with quoted Quickwit syntax', () => { + const result = renderAdHocFilters( + { service_name: 'text' }, + [{ + key: 'service_name', + operator: '=', + value: 'auth-api', + }] + ); + + expect(result).toBe('service_name:"auth-api"'); + }); + + it('escapes special characters in unquoted term filters', () => { + const result = addAddHocFilter('', { + key: 'attributes.grpc_message', + operator: 'term', + value: 'error:foo', + }); + + expect(result).toBe('attributes.grpc_message:error\\:foo'); + }); + + it('applies prior filters when loading tag values', async () => { + const getTerms = jest.fn(() => from([[]])); + + await (BaseQuickwitDataSource.prototype as any).getTagValues.call( + { + fieldTypes: {}, + addAdHocFilters: BaseQuickwitDataSource.prototype.addAdHocFilters, + getTerms, + }, + { + key: 'status', + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(getTerms).toHaveBeenCalledWith( + { field: 'status', query: 'service:"frontend"' }, + undefined + ); + }); + }); }); diff --git a/src/datasource/base.ts b/src/datasource/base.ts index d7b132c..f91446d 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -3,7 +3,6 @@ import { Observable, lastValueFrom, from, of, map, mergeMap } from 'rxjs'; import { AbstractQuery, AdHocVariableFilter, - AppEvents, CoreApp, DataFrame, DataQueryRequest, @@ -20,7 +19,6 @@ import { import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery, FieldCapabilitiesResponse } from '@/types'; import { DataSourceWithBackend, - getAppEvents, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; import { QuickwitOptions } from 'quickwit'; @@ -31,7 +29,7 @@ import { isMetricAggregationWithField } from 'components/QueryEditor/MetricAggre import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregationsEditor/utils'; import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations'; import ElasticsearchLanguageProvider from 'LanguageProvider'; -import { fieldTypeMap, hasWhiteSpace } from 'utils'; +import { fieldTypeMap, hasWhiteSpace, isSimpleToken } from 'utils'; import { addAddHocFilter } from 'modifyQuery'; import { getQueryResponseProcessor } from 'datasource/processResponse'; @@ -67,13 +65,8 @@ export class BaseQuickwitDataSource }; languageProvider: ElasticsearchLanguageProvider; // Populated lazily by getFields(). Used by modifyQuery to pick an operator - // that's safe for the field's type (phrase queries require positions indexed, - // which Quickwit text fields don't have by default). + // that matches text field semantics. private fieldTypes: Record = {}; - // Tracks (key|value|operator) triples we've already warned about on this - // datasource instance so the same filter doesn't spam a toast on every - // dashboard re-render. - private warnedAdHocFilters: Set = new Set(); getFieldType(name: string): string | undefined { return this.fieldTypes[name]; @@ -184,23 +177,9 @@ export class BaseQuickwitDataSource const fieldType = this.fieldTypes[key]; const isText = fieldType === 'text'; - // Text field + value with whitespace: no single filter can represent this. - // Quickwit text fields lack positions so phrase fails, and `term` can't - // express whitespace. Warn the user and abort rather than add a broken or - // approximate filter. - if (isText && hasWhiteSpace(rawValue)) { - getAppEvents().publish({ - type: AppEvents.alertWarning.name, - payload: [ - `Cannot filter on "${key}"`, - `Quickwit text fields don't index positions by default, so "${rawValue}" can't be filtered as a phrase. Add this filter manually or edit the Lucene query.`, - ], - }); - return query; - } - const value = rawValue; - const operator = isText && value !== '' + const useTermOperator = isText && value !== '' && isSimpleToken(value); + const operator = useTermOperator ? (negate ? 'not term' : 'term') : (negate ? '!=' : '='); @@ -314,7 +293,7 @@ export class BaseQuickwitDataSource /** * Get tag keys for adhoc filters */ - getTagKeys(options: any) { + getTagKeys(options: any = {}) { const fields = this.getFields({aggregatable:true, range: options.timeRange}) return lastValueFrom(fields, {defaultValue:[]}); } @@ -323,7 +302,8 @@ export class BaseQuickwitDataSource * Get tag values for adhoc filters */ getTagValues(options: any) { - const terms = this.getTerms({ field: options.key }, options.timeRange) + const query = this.addAdHocFilters('', options.filters); + const terms = this.getTerms({ field: options.key, query }, options.timeRange) return lastValueFrom(terms, {defaultValue:[]}); } @@ -478,30 +458,10 @@ export class BaseQuickwitDataSource const isText = fieldType === 'text'; const value = filter.value ?? ''; - // Text field + whitespace value + phrase operator would emit a phrase query, - // which fails on Quickwit text fields without positions indexed. Skip the - // filter — better a no-op than a broken panel. Fire a toast once per - // (key|value|operator) so the user knows why it's not applying, without - // spamming on every dashboard render. - if (isText && hasWhiteSpace(value) && (filter.operator === '=' || filter.operator === '!=')) { - const dedupKey = `${filter.key}|${value}|${filter.operator}`; - if (!this.warnedAdHocFilters.has(dedupKey)) { - this.warnedAdHocFilters.add(dedupKey); - getAppEvents().publish({ - type: AppEvents.alertWarning.name, - payload: [ - `Filter on "${filter.key}" was skipped`, - `Quickwit text fields don't index positions by default, so "${value}" can't be filtered as a phrase. Edit the Lucene query directly for this case.`, - ], - }); - } - return; - } - - // For text fields with single-token values, upgrade '=' to 'term' (and + // For text fields with simple token values, upgrade '=' to 'term' (and // '!=' to 'not term') so the emitted query doesn't require positions. let effective = filter; - if (isText && !hasWhiteSpace(value)) { + if (isText && isSimpleToken(value)) { if (filter.operator === '=') { effective = { ...filter, operator: 'term' }; } else if (filter.operator === '!=') { diff --git a/src/hooks/useFields.ts b/src/hooks/useFields.ts index b4f2192..06579ce 100644 --- a/src/hooks/useFields.ts +++ b/src/hooks/useFields.ts @@ -6,6 +6,7 @@ import { isBucketAggregationType } from '../components/QueryEditor/BucketAggrega import { useDatasource, useRange } from '../components/QueryEditor/ElasticsearchQueryContext'; import { isMetricAggregationType } from '../components/QueryEditor/MetricAggregationsEditor/aggregations'; import { MetricAggregationType, BucketAggregationType } from '../types'; +import { fuzzySearchMatch, fuzzySearchSort } from '@/utils'; type AggregationType = BucketAggregationType | MetricAggregationType; @@ -46,7 +47,7 @@ const toSelectableValue = ({ text }: MetricFindValue): SelectableValue = value: text, }); -type MatchType = 'contains' | 'startsWith' +type MatchType = 'contains' | 'startsWith' | 'fuzzy' /** * Returns a function to query the configured datasource for autocomplete values for the specified aggregation type or data types. @@ -68,13 +69,18 @@ export const useFields = (type: AggregationType | string[], matchType: MatchType rawFields = await lastValueFrom(datasource.getFields({aggregatable:true, type:filter, range:range})); } - return rawFields + const fields = rawFields .filter(({ text }) => { if (q === undefined) { return true; } + if (matchType === 'fuzzy') { + return fuzzySearchMatch(text, q); + } return matchType === 'contains' ? text.includes(q) : text.startsWith(q) - }) + }); + + return (matchType === 'fuzzy' ? fuzzySearchSort(fields, ({ text }) => text, q) : fields) .map(toSelectableValue); }; }; diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index 5b79c9e..615c102 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -26,6 +26,7 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str */ const key = escapeFilter(filter.key); const value = escapeFilterValue(filter.value); + const termValue = escapeFilter(filter.value); let addHocFilter = ''; switch (filter.operator) { case '=~': @@ -41,10 +42,10 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str addHocFilter = `${key}:<${value}`; break; case 'term': - addHocFilter = `${key}:${value}`; + addHocFilter = `${key}:${termValue}`; break; case 'not term': - addHocFilter = `-${key}:${value}`; + addHocFilter = `-${key}:${termValue}`; break; case 'exists': addHocFilter = `${key}:*`; diff --git a/src/utils/index.test.ts b/src/utils/index.test.ts index 1bcec2c..4552623 100644 --- a/src/utils/index.test.ts +++ b/src/utils/index.test.ts @@ -1,4 +1,4 @@ -import { extractJsonPayload } from "utils"; +import { extractJsonPayload, fuzzySearchMatch, fuzzySearchSort, isSimpleToken } from "utils"; describe('Test utils.extractJsonPayload', () => { it('Extract valid JSON', () => { @@ -18,3 +18,37 @@ describe('Test utils.extractJsonPayload', () => { expect(result).toEqual(null); }); }); + +describe('fuzzy search helpers', () => { + it('matches field names using token abbreviations', () => { + expect(fuzzySearchMatch('attributes.grpc_message', 'grpc msg')).toBe(true); + }); + + it('matches values with missing characters', () => { + expect(fuzzySearchMatch('Error invalid token', 'err invld')).toBe(true); + }); + + it('sorts stronger matches first', () => { + const result = fuzzySearchSort( + ['body.message', 'attributes.grpc_message', 'attributes.http_status'], + (value) => value, + 'grpc msg' + ); + + expect(result[0]).toBe('attributes.grpc_message'); + }); + + it('does not match unrelated values', () => { + expect(fuzzySearchMatch('attributes.grpc_message', 'status code')).toBe(false); + }); +}); + +describe('isSimpleToken', () => { + it('accepts analyzer-friendly bare tokens', () => { + expect(isSimpleToken('auth_api_1')).toBe(true); + }); + + it('rejects punctuated values that should be quoted as phrases', () => { + expect(isSimpleToken('auth-api')).toBe(false); + }); +}); diff --git a/src/utils/index.ts b/src/utils/index.ts index 518cf43..ace2cb4 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -136,3 +136,127 @@ export const isSet = (v: string) => v !== '' && v !== undefined && v !== null; export const hasWhiteSpace = (s: string) => /\s/g.test(s); +export const isSimpleToken = (s: string) => /^[A-Za-z0-9_]+$/.test(s); + +const normalizeSearchText = (value: string) => value.toLowerCase(); + +const searchTokens = (value: string) => + normalizeSearchText(value) + .split(/[^a-z0-9]+/g) + .filter(Boolean); + +const isSubsequence = (needle: string, haystack: string) => { + let needleIndex = 0; + for (let haystackIndex = 0; haystackIndex < haystack.length && needleIndex < needle.length; haystackIndex++) { + if (needle[needleIndex] === haystack[haystackIndex]) { + needleIndex++; + } + } + return needleIndex === needle.length; +}; + +const levenshteinDistance = (a: string, b: string, maxDistance: number) => { + if (Math.abs(a.length - b.length) > maxDistance) { + return maxDistance + 1; + } + + let previous = Array.from({ length: b.length + 1 }, (_, index) => index); + for (let i = 1; i <= a.length; i++) { + const current = [i]; + let rowMin = current[0]; + for (let j = 1; j <= b.length; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + current[j] = Math.min( + current[j - 1] + 1, + previous[j] + 1, + previous[j - 1] + cost + ); + rowMin = Math.min(rowMin, current[j]); + } + if (rowMin > maxDistance) { + return maxDistance + 1; + } + previous = current; + } + + return previous[b.length]; +}; + +const fuzzyTokenScore = (candidateTokens: string[], compactCandidate: string, token: string): number | null => { + if (!token) { + return 0; + } + + let best: number | null = null; + for (const candidateToken of candidateTokens) { + let score: number | null = null; + if (candidateToken === token) { + score = 0; + } else if (candidateToken.startsWith(token)) { + score = 10 + candidateToken.length - token.length; + } else if (candidateToken.includes(token)) { + score = 25 + candidateToken.indexOf(token); + } else if (isSubsequence(token, candidateToken)) { + score = 50 + candidateToken.length - token.length; + } else if (token.length >= 3) { + const maxDistance = Math.max(1, Math.floor(token.length / 3)); + const distance = levenshteinDistance(token, candidateToken, maxDistance); + if (distance <= maxDistance) { + score = 80 + distance * 10 + Math.abs(candidateToken.length - token.length); + } + } + + if (score !== null && (best === null || score < best)) { + best = score; + } + } + + if (best === null && compactCandidate.includes(token)) { + return 35 + compactCandidate.indexOf(token); + } + + return best; +}; + +export const fuzzySearchScore = (text: string, query?: string): number | null => { + const trimmedQuery = query?.trim(); + if (!trimmedQuery) { + return 0; + } + + const normalizedText = normalizeSearchText(text); + const normalizedQuery = normalizeSearchText(trimmedQuery); + if (normalizedText === normalizedQuery) { + return 0; + } + if (normalizedText.startsWith(normalizedQuery)) { + return 5 + normalizedText.length - normalizedQuery.length; + } + if (normalizedText.includes(normalizedQuery)) { + return 15 + normalizedText.indexOf(normalizedQuery); + } + + const candidateTokens = searchTokens(text); + const compactCandidate = candidateTokens.join(''); + const queryTokens = searchTokens(trimmedQuery); + let total = 0; + for (const token of queryTokens) { + const score = fuzzyTokenScore(candidateTokens, compactCandidate, token); + if (score === null) { + return null; + } + total += score; + } + + return total; +}; + +export const fuzzySearchMatch = (text: string, query?: string) => fuzzySearchScore(text, query) !== null; + +export function fuzzySearchSort(items: T[], getText: (item: T) => string, query?: string): T[] { + return items + .map((item, index) => ({ item, index, score: fuzzySearchScore(getText(item), query) })) + .filter((item): item is { item: T; index: number; score: number } => item.score !== null) + .sort((a, b) => a.score - b.score || a.index - b.index) + .map(({ item }) => item); +} From 506d054fa5681d3b743532861539220d6954e29d Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Fri, 24 Apr 2026 23:30:05 -0400 Subject: [PATCH 08/14] Add tests capturing current addAddHocFilter behavior Documents how the = operator wraps array values like ["paperclip"] in quotes, producing phrase queries that don't match indexed array elements. Tests cover all operator types and edge cases. Refs quickwit-oss/quickwit-datasource#179 --- src/modifyQuery.test.ts | 137 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 src/modifyQuery.test.ts diff --git a/src/modifyQuery.test.ts b/src/modifyQuery.test.ts new file mode 100644 index 0000000..6cb77c2 --- /dev/null +++ b/src/modifyQuery.test.ts @@ -0,0 +1,137 @@ +import { addAddHocFilter } from './modifyQuery'; + +describe('addAddHocFilter', () => { + describe('current behavior with array values', () => { + it('wraps equality filter value in quotes (phrase query)', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["paperclip"]', + }); + // Current behavior: generates a phrase query with the stringified array + expect(result).toBe('attributes.tags:"[\\"paperclip\\"]"'); + }); + + it('wraps negated equality filter value in quotes', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '!=', + value: '["paperclip"]', + }); + expect(result).toBe('-attributes.tags:"[\\"paperclip\\"]"'); + }); + + it('term operator produces unquoted query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: 'term', + value: 'paperclip', + }); + expect(result).toBe('attributes.tags:paperclip'); + }); + + it('not term operator produces negated unquoted query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: 'not term', + value: 'paperclip', + }); + expect(result).toBe('-attributes.tags:paperclip'); + }); + }); + + describe('scalar value filters', () => { + it('equality on simple string value', () => { + const result = addAddHocFilter('', { + key: 'attributes.controller', + operator: '=', + value: 'BlogController', + }); + expect(result).toBe('attributes.controller:"BlogController"'); + }); + + it('appends to existing query with AND', () => { + const result = addAddHocFilter('status:200', { + key: 'attributes.controller', + operator: '=', + value: 'BlogController', + }); + expect(result).toBe('status:200 AND attributes.controller:"BlogController"'); + }); + + it('exists operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: 'exists', + value: '', + }); + expect(result).toBe('attributes.tags:*'); + }); + + it('not exists operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: 'not exists', + value: '', + }); + expect(result).toBe('-attributes.tags:*'); + }); + + it('regex operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.controller', + operator: '=~', + value: 'Blog.*', + }); + expect(result).toBe('attributes.controller:/Blog.*/'); + }); + + it('greater than operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.duration', + operator: '>', + value: '100', + }); + expect(result).toBe('attributes.duration:>100'); + }); + + it('less than operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.duration', + operator: '<', + value: '100', + }); + expect(result).toBe('attributes.duration:<100'); + }); + }); + + describe('edge cases', () => { + it('returns query unchanged when key is empty', () => { + const result = addAddHocFilter('existing', { + key: '', + operator: '=', + value: 'test', + }); + expect(result).toBe('existing'); + }); + + it('returns query unchanged when value is empty for non-exists operators', () => { + const result = addAddHocFilter('existing', { + key: 'field', + operator: '=', + value: '', + }); + expect(result).toBe('existing'); + }); + + it('handles multi-element array value with equality', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["paperclip","stapler"]', + }); + // Current behavior: entire stringified array becomes the phrase + expect(result).toBe('attributes.tags:"[\\"paperclip\\",\\"stapler\\"]"'); + }); + }); +}); From d671b1ab711399f1a333d11319edbdd481fbd2d1 Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Fri, 24 Apr 2026 23:34:16 -0400 Subject: [PATCH 09/14] Unwrap JSON array values in ad-hoc equality filters When Grafana passes a stringified array like ["paperclip"] as a filter value, detect it as JSON array and generate term queries on the individual elements instead of a phrase query on the literal string. Single element: attributes.tags:paperclip Multiple elements: attributes.tags:paperclip OR attributes.tags:stapler Empty array: no-op Fixes quickwit-oss/quickwit-datasource#179 --- src/modifyQuery.test.ts | 51 ++++++++++++++++++++++++++++++----------- src/modifyQuery.ts | 26 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/modifyQuery.test.ts b/src/modifyQuery.test.ts index 6cb77c2..cc8e7a5 100644 --- a/src/modifyQuery.test.ts +++ b/src/modifyQuery.test.ts @@ -1,27 +1,53 @@ import { addAddHocFilter } from './modifyQuery'; describe('addAddHocFilter', () => { - describe('current behavior with array values', () => { - it('wraps equality filter value in quotes (phrase query)', () => { + describe('array values', () => { + it('unwraps single-element array into a term query', () => { const result = addAddHocFilter('', { key: 'attributes.tags', operator: '=', value: '["paperclip"]', }); - // Current behavior: generates a phrase query with the stringified array - expect(result).toBe('attributes.tags:"[\\"paperclip\\"]"'); + expect(result).toBe('attributes.tags:paperclip'); }); - it('wraps negated equality filter value in quotes', () => { + it('unwraps multi-element array into OR of term queries', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["paperclip","stapler"]', + }); + expect(result).toBe('attributes.tags:paperclip OR attributes.tags:stapler'); + }); + + it('negated array produces negated term queries', () => { const result = addAddHocFilter('', { key: 'attributes.tags', operator: '!=', value: '["paperclip"]', }); - expect(result).toBe('-attributes.tags:"[\\"paperclip\\"]"'); + expect(result).toBe('-attributes.tags:paperclip'); }); - it('term operator produces unquoted query', () => { + it('appends array filter to existing query with AND', () => { + const result = addAddHocFilter('status:200', { + key: 'attributes.tags', + operator: '=', + value: '["paperclip"]', + }); + expect(result).toBe('status:200 AND attributes.tags:paperclip'); + }); + + it('passes through non-array bracket strings unchanged', () => { + const result = addAddHocFilter('', { + key: 'attributes.message', + operator: '=', + value: '[not json', + }); + expect(result).toBe('attributes.message:"[not json"'); + }); + + it('term operator still produces unquoted query', () => { const result = addAddHocFilter('', { key: 'attributes.tags', operator: 'term', @@ -30,7 +56,7 @@ describe('addAddHocFilter', () => { expect(result).toBe('attributes.tags:paperclip'); }); - it('not term operator produces negated unquoted query', () => { + it('not term operator still produces negated unquoted query', () => { const result = addAddHocFilter('', { key: 'attributes.tags', operator: 'not term', @@ -124,14 +150,13 @@ describe('addAddHocFilter', () => { expect(result).toBe('existing'); }); - it('handles multi-element array value with equality', () => { - const result = addAddHocFilter('', { + it('treats empty JSON array as no-op', () => { + const result = addAddHocFilter('existing', { key: 'attributes.tags', operator: '=', - value: '["paperclip","stapler"]', + value: '[]', }); - // Current behavior: entire stringified array becomes the phrase - expect(result).toBe('attributes.tags:"[\\"paperclip\\",\\"stapler\\"]"'); + expect(result).toBe('existing'); }); }); }); diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index 615c102..50cf5e4 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -1,6 +1,21 @@ import { escapeFilter, escapeFilterValue, concatenate, LuceneQuery } from 'utils/lucene'; import { AdHocVariableFilter } from '@grafana/data'; +function tryParseJsonArray(value: string): string[] | null { + if (!value.startsWith('[')) { + return null; + } + try { + const parsed = JSON.parse(value); + if (Array.isArray(parsed) && parsed.every((el) => typeof el === 'string')) { + return parsed; + } + } catch { + // not valid JSON + } + return null; +} + /** * Adds a label:"value" expression to the query. */ @@ -18,6 +33,17 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str const equalityFilters = ['=', '!=']; if (equalityFilters.includes(filter.operator)) { + const arrayElements = tryParseJsonArray(filter.value); + if (arrayElements !== null) { + if (arrayElements.length === 0) { + return query; + } + const modifier = filter.operator === '=' ? '' : '-'; + const key = escapeFilter(filter.key); + const termFilters = arrayElements.map((el) => `${modifier}${key}:${escapeFilterValue(el)}`); + const combined = termFilters.join(' OR '); + return concatenate(query, combined, 'AND'); + } return LuceneQuery.parse(query).addFilter(filter.key, filter.value, filter.operator === '=' ? '' : '-').toString(); } /** From 6de328c5d03f2b4e8f81393b74deb67367be16cf Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Fri, 24 Apr 2026 23:37:59 -0400 Subject: [PATCH 10/14] Use IN set query for multi-element arrays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-element arrays still produce a simple term query. Multi-element arrays now produce field:IN ["a" "b"] which maps to tantivy's TermSetQuery — cleaner than chaining OR clauses. Refs quickwit-oss/quickwit-datasource#179 --- src/modifyQuery.test.ts | 15 ++++++++++++--- src/modifyQuery.ts | 8 +++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/modifyQuery.test.ts b/src/modifyQuery.test.ts index cc8e7a5..ee19954 100644 --- a/src/modifyQuery.test.ts +++ b/src/modifyQuery.test.ts @@ -11,16 +11,16 @@ describe('addAddHocFilter', () => { expect(result).toBe('attributes.tags:paperclip'); }); - it('unwraps multi-element array into OR of term queries', () => { + it('unwraps multi-element array into IN set query', () => { const result = addAddHocFilter('', { key: 'attributes.tags', operator: '=', value: '["paperclip","stapler"]', }); - expect(result).toBe('attributes.tags:paperclip OR attributes.tags:stapler'); + expect(result).toBe('attributes.tags:IN ["paperclip" "stapler"]'); }); - it('negated array produces negated term queries', () => { + it('negated single-element array produces negated term query', () => { const result = addAddHocFilter('', { key: 'attributes.tags', operator: '!=', @@ -29,6 +29,15 @@ describe('addAddHocFilter', () => { expect(result).toBe('-attributes.tags:paperclip'); }); + it('negated multi-element array produces negated IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '!=', + value: '["paperclip","stapler"]', + }); + expect(result).toBe('-attributes.tags:IN ["paperclip" "stapler"]'); + }); + it('appends array filter to existing query with AND', () => { const result = addAddHocFilter('status:200', { key: 'attributes.tags', diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index 50cf5e4..97a6945 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -40,9 +40,11 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str } const modifier = filter.operator === '=' ? '' : '-'; const key = escapeFilter(filter.key); - const termFilters = arrayElements.map((el) => `${modifier}${key}:${escapeFilterValue(el)}`); - const combined = termFilters.join(' OR '); - return concatenate(query, combined, 'AND'); + if (arrayElements.length === 1) { + return concatenate(query, `${modifier}${key}:${escapeFilterValue(arrayElements[0])}`, 'AND'); + } + const terms = arrayElements.map((el) => `"${escapeFilterValue(el)}"`).join(' '); + return concatenate(query, `${modifier}${key}:IN [${terms}]`, 'AND'); } return LuceneQuery.parse(query).addFilter(filter.key, filter.value, filter.operator === '=' ? '' : '-').toString(); } From ccef7f2856958069a0cfdbad9ada475e557e05a3 Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Fri, 24 Apr 2026 23:39:44 -0400 Subject: [PATCH 11/14] Add comment explaining array filter semantics Tantivy indexes array elements as individual terms with no way to match on array structure. Document why IN (match any) is used for multi-element arrays. --- src/modifyQuery.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index 97a6945..e2ae74f 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -33,6 +33,11 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str const equalityFilters = ['=', '!=']; if (equalityFilters.includes(filter.operator)) { + // Grafana stringifies array values (e.g. ["paperclip","stapler"]) before + // passing them as filter values. Tantivy indexes array elements as + // individual terms — there's no way to match on array length, order, or + // exact composition. For multi-element arrays we use IN (match any), + // which is the most useful behavior for log exploration filters. const arrayElements = tryParseJsonArray(filter.value); if (arrayElements !== null) { if (arrayElements.length === 0) { From 630ed79ea7ae5652878896a40a91c25985742826 Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Mon, 27 Apr 2026 10:43:06 -0400 Subject: [PATCH 12/14] Quote single-element array values to handle spaces and colons Values like ["foo bar"] or ["foo:bar"] produced broken queries (e.g. field:foo bar) because the single-element path emitted bare terms. Now all array element values are quoted as phrase queries, consistent with the multi-element IN path. --- src/modifyQuery.test.ts | 46 ++++++++++++++++++++++++++++++++++++----- src/modifyQuery.ts | 2 +- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/modifyQuery.test.ts b/src/modifyQuery.test.ts index ee19954..85da34b 100644 --- a/src/modifyQuery.test.ts +++ b/src/modifyQuery.test.ts @@ -2,13 +2,13 @@ import { addAddHocFilter } from './modifyQuery'; describe('addAddHocFilter', () => { describe('array values', () => { - it('unwraps single-element array into a term query', () => { + it('unwraps single-element array into a phrase query', () => { const result = addAddHocFilter('', { key: 'attributes.tags', operator: '=', value: '["paperclip"]', }); - expect(result).toBe('attributes.tags:paperclip'); + expect(result).toBe('attributes.tags:"paperclip"'); }); it('unwraps multi-element array into IN set query', () => { @@ -20,13 +20,13 @@ describe('addAddHocFilter', () => { expect(result).toBe('attributes.tags:IN ["paperclip" "stapler"]'); }); - it('negated single-element array produces negated term query', () => { + it('negated single-element array produces negated phrase query', () => { const result = addAddHocFilter('', { key: 'attributes.tags', operator: '!=', value: '["paperclip"]', }); - expect(result).toBe('-attributes.tags:paperclip'); + expect(result).toBe('-attributes.tags:"paperclip"'); }); it('negated multi-element array produces negated IN set query', () => { @@ -44,7 +44,43 @@ describe('addAddHocFilter', () => { operator: '=', value: '["paperclip"]', }); - expect(result).toBe('status:200 AND attributes.tags:paperclip'); + expect(result).toBe('status:200 AND attributes.tags:"paperclip"'); + }); + + it('handles single-element array with spaces in value', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["foo bar"]', + }); + expect(result).toBe('attributes.tags:"foo bar"'); + }); + + it('handles single-element array with colons in value', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["foo:bar"]', + }); + expect(result).toBe('attributes.tags:"foo:bar"'); + }); + + it('handles multi-element array with spaces in values', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["foo bar","baz qux"]', + }); + expect(result).toBe('attributes.tags:IN ["foo bar" "baz qux"]'); + }); + + it('handles array values containing double quotes', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["say \\"hello\\""]', + }); + expect(result).toBe('attributes.tags:"say \\"hello\\""'); }); it('passes through non-array bracket strings unchanged', () => { diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index e2ae74f..3141735 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -46,7 +46,7 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str const modifier = filter.operator === '=' ? '' : '-'; const key = escapeFilter(filter.key); if (arrayElements.length === 1) { - return concatenate(query, `${modifier}${key}:${escapeFilterValue(arrayElements[0])}`, 'AND'); + return concatenate(query, `${modifier}${key}:"${escapeFilterValue(arrayElements[0])}"`, 'AND'); } const terms = arrayElements.map((el) => `"${escapeFilterValue(el)}"`).join(' '); return concatenate(query, `${modifier}${key}:IN [${terms}]`, 'AND'); From 239b21215944e472cdfeaa3e98b1dae8287631fb Mon Sep 17 00:00:00 2001 From: Patrik Cyvoct Date: Tue, 28 Apr 2026 23:33:06 +0200 Subject: [PATCH 13/14] fix: filters and configurable autocomplete chains - render Quickwit-safe filters for text phrases, simple text terms, JSON arrays, numeric arrays, boolean arrays, and scalar numeric/boolean values - fix quick-filter toggle behavior so filter-in/filter-out matches by operator and replaces opposite filters instead of adding duplicates - avoid mutating query filter objects when adding or toggling quick filters - add configurable filter autocomplete limit, defaulting to 1000 and supporting 0 for no terms limit - apply autocomplete limit to tag values and field autocomplete requests - add filter chain mode datasource option: no chain, sampled chain, or full chain - dedupe tag keys when fields expose multiple capabilities - improve template multi-value interpolation for Quickwit by inferring field context and avoiding bare IN fallbacks - add focused tests for filter rendering, quick-filter toggles, autocomplete limits, chain modes, and template interpolation Signed-off-by: Patrik Cyvoct --- src/QueryBuilder/elastic.test.ts | 25 ++ src/QueryBuilder/elastic.ts | 3 +- src/configuration/ConfigEditor.tsx | 53 ++- src/configuration/utils.ts | 7 + src/datasource/base.test.ts | 629 +++++++++++++++++++++++------ src/datasource/base.ts | 329 +++++++++++++-- src/modifyQuery.test.ts | 90 +++++ src/modifyQuery.ts | 40 +- src/quickwit.ts | 6 + 9 files changed, 1005 insertions(+), 177 deletions(-) create mode 100644 src/QueryBuilder/elastic.test.ts diff --git a/src/QueryBuilder/elastic.test.ts b/src/QueryBuilder/elastic.test.ts new file mode 100644 index 0000000..28ebbed --- /dev/null +++ b/src/QueryBuilder/elastic.test.ts @@ -0,0 +1,25 @@ +import { getDataQuery } from './elastic'; + +describe('getDataQuery', () => { + it('uses the requested terms size for autocomplete queries', () => { + const query = getDataQuery({ field: 'status', size: 250 }, 'getTerms'); + + expect(query.bucketAggs?.[0].settings).toEqual( + expect.objectContaining({ + size: '250', + shard_size: '250', + }) + ); + }); + + it('keeps zero as no terms limit', () => { + const query = getDataQuery({ field: 'status', size: 0 }, 'getTerms'); + + expect(query.bucketAggs?.[0].settings).toEqual( + expect.objectContaining({ + size: '0', + shard_size: '0', + }) + ); + }); +}); diff --git a/src/QueryBuilder/elastic.ts b/src/QueryBuilder/elastic.ts index 907f66e..d6df917 100644 --- a/src/QueryBuilder/elastic.ts +++ b/src/QueryBuilder/elastic.ts @@ -58,7 +58,8 @@ export function getDataQuery(queryDef: TermsQuery, refId: string): Elasticsearch const bucketAggs: BucketAggregation[] = []; if (queryDef.field) { - bucketAggs.push(getTermsAgg(queryDef.field, 100, 100, orderBy, order)) + const size = queryDef.size ?? 100; + bucketAggs.push(getTermsAgg(queryDef.field, size, size, orderBy, order)) } return { diff --git a/src/configuration/ConfigEditor.tsx b/src/configuration/ConfigEditor.tsx index 511902d..5bac6d8 100644 --- a/src/configuration/ConfigEditor.tsx +++ b/src/configuration/ConfigEditor.tsx @@ -1,7 +1,7 @@ import React, { useCallback } from 'react'; -import { DataSourceHttpSettings, Input, InlineField, FieldSet } from '@grafana/ui'; -import { DataSourcePluginOptionsEditorProps, DataSourceSettings } from '@grafana/data'; -import { QuickwitOptions } from '../quickwit'; +import { DataSourceHttpSettings, Input, InlineField, FieldSet, RadioButtonGroup } from '@grafana/ui'; +import { DataSourcePluginOptionsEditorProps, DataSourceSettings, SelectableValue } from '@grafana/data'; +import { FilterAutocompleteChainMode, QuickwitOptions } from '../quickwit'; import { coerceOptions } from './utils'; import { Divider } from '../components/Divider'; import { DataLinks } from './DataLinks'; @@ -9,6 +9,12 @@ import _ from 'lodash'; interface Props extends DataSourcePluginOptionsEditorProps {} +const filterChainModeOptions: Array> = [ + { label: 'No chain', value: 'none' }, + { label: 'Sample', value: 'sample' }, + { label: 'Full', value: 'full' }, +]; + export const ConfigEditor = (props: Props) => { const { options: originalOptions, onOptionsChange } = props; const options = coerceOptions(originalOptions); @@ -103,6 +109,47 @@ export const QuickwitDetails = ({ value, onChange }: DetailsProps) => { width={40} /> + + + onChange({ + ...value, + jsonData: { ...value.jsonData, filterAutocompleteLimit: event.currentTarget.value }, + }) + } + placeholder="1000" + width={40} + /> + + + + onChange({ + ...value, + jsonData: { + ...value.jsonData, + filterAutocompleteChainMode: mode, + filterAutocompleteUseFilterChains: mode !== 'none', + }, + }) + } + /> + diff --git a/src/configuration/utils.ts b/src/configuration/utils.ts index 748c2d0..1a7c47d 100644 --- a/src/configuration/utils.ts +++ b/src/configuration/utils.ts @@ -4,12 +4,19 @@ import { QuickwitOptions } from 'quickwit'; export const coerceOptions = ( options: DataSourceSettings ): DataSourceSettings => { + const filterAutocompleteChainMode = + options.jsonData.filterAutocompleteChainMode ?? + (options.jsonData.filterAutocompleteUseFilterChains === false ? 'none' : 'sample'); + return { ...options, jsonData: { ...options.jsonData, logMessageField: options.jsonData.logMessageField || '', logLevelField: options.jsonData.logLevelField || '', + filterAutocompleteLimit: options.jsonData.filterAutocompleteLimit ?? '1000', + filterAutocompleteChainMode, + filterAutocompleteUseFilterChains: filterAutocompleteChainMode !== 'none', }, }; }; diff --git a/src/datasource/base.test.ts b/src/datasource/base.test.ts index 22f1d43..a4b9eff 100644 --- a/src/datasource/base.test.ts +++ b/src/datasource/base.test.ts @@ -3,7 +3,13 @@ import { from } from 'rxjs'; import { addAddHocFilter } from '../modifyQuery'; import { ElasticsearchQuery } from '../types'; -import { BaseQuickwitDataSource, formatQuery, luceneEscape } from './base'; +import { + BaseQuickwitDataSource, + formatQuery, + luceneEscape, + parseFilterAutocompleteChainMode, + parseFilterAutocompleteLimit, +} from './base'; describe('BaseQuickwitDataSource', () => { describe('luceneEscape', () => { @@ -29,153 +35,164 @@ describe('BaseQuickwitDataSource', () => { }); describe('formatQuery', () => { + describe('String values', () => { + it('should return escaped string for simple string values', () => { + const result = formatQuery('simple_value', { id: 'test_var' }); + expect(result).toBe('simple_value'); + }); - describe('String values', () => { - it('should return escaped string for simple string values', () => { - const result = formatQuery('simple_value', { id: 'test_var' }); - expect(result).toBe('simple_value'); - }); + it('should escape special characters in string values', () => { + const result = formatQuery('value+with-special:chars', { id: 'test_var' }); + expect(result).toBe('value\\+with\\-special\\:chars'); + }); - it('should escape special characters in string values', () => { - const result = formatQuery('value+with-special:chars', { id: 'test_var' }); - expect(result).toBe('value\\+with\\-special\\:chars'); + it('should not escape numeric strings', () => { + const result = formatQuery('123', { id: 'test_var' }); + expect(result).toBe('123'); + }); }); - it('should not escape numeric strings', () => { - const result = formatQuery('123', { id: 'test_var' }); - expect(result).toBe('123'); - }); - }); + describe('Array values with valid field configuration', () => { + const validVariable = { + id: 'test_var', + query: '{"field": "status"}' + }; - describe('Array values with valid field configuration', () => { - const validVariable = { - id: 'test_var', - query: '{"field": "status"}' - }; + it('should format array values with field-specific OR syntax', () => { + const result = formatQuery(['error', 'warning'], validVariable); + expect(result).toBe('"error" OR status:"warning"'); + }); - it('should format array values with field-specific OR syntax', () => { - const result = formatQuery(['error', 'warning'], validVariable); - expect(result).toBe('"error" OR status:"warning"'); - }); + it('should handle single-item arrays', () => { + const result = formatQuery(['error'], validVariable); + expect(result).toBe('"error"'); + }); - it('should handle single-item arrays', () => { - const result = formatQuery(['error'], validVariable); - expect(result).toBe('"error"'); + it('should escape special characters in array values', () => { + const result = formatQuery(['error+critical', 'warning:high'], validVariable); + expect(result).toBe('"error\\+critical" OR status:"warning\\:high"'); + }); }); - it('should escape special characters in array values', () => { - const result = formatQuery(['error+critical', 'warning:high'], validVariable); - expect(result).toBe('"error\\+critical" OR status:"warning\\:high"'); - }); - }); + describe('Array values without field configuration', () => { + it('should use OR syntax when variable.query is undefined', () => { + const variable = { id: 'test_var' }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); - describe('Array values without field configuration', () => { - it('should use IN syntax when variable.query is undefined', () => { - const variable = { id: 'test_var' }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); - }); + it('should use OR syntax when variable.query is null', () => { + const variable = { id: 'test_var', query: null }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); - it('should use IN syntax when variable.query is null', () => { - const variable = { id: 'test_var', query: null }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); - }); + it('should use OR syntax when variable.query contains invalid JSON', () => { + const variable = { id: 'test_var', query: 'not valid json' }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); - it('should use IN syntax when variable.query contains invalid JSON', () => { - const variable = { id: 'test_var', query: 'not valid json' }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); - }); + it('should use OR syntax when variable.query is valid JSON but missing field', () => { + const variable = { id: 'test_var', query: '{"other": "value"}' }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); - it('should use IN syntax when variable.query is valid JSON but missing field', () => { - const variable = { id: 'test_var', query: '{"other": "value"}' }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); - }); + it('should use OR syntax when field is not a string', () => { + const variable = { id: 'test_var', query: '{"field": 123}' }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); - it('should use IN syntax when field is not a string', () => { - const variable = { id: 'test_var', query: '{"field": 123}' }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); - }); - }); + it('should infer field-specific OR syntax from the query string', () => { + const variable = { id: 'levels' }; + const result = formatQuery(['error', 'warning'], variable, 'severity_text:$levels'); + expect(result).toBe('"error" OR severity_text:"warning"'); + }); - describe('Empty arrays', () => { - it('should return __empty__ for empty arrays', () => { - const variable = { id: 'test_var', query: '{"field": "status"}' }; - const result = formatQuery([], variable); - expect(result).toBe('__empty__'); + it('should infer field-specific OR syntax from braced variables in the query string', () => { + const variable = { id: 'services' }; + const result = formatQuery(['web', 'api'], variable, 'resource_attributes.service.name:${services}'); + expect(result).toBe('"web" OR resource_attributes.service.name:"api"'); + }); }); - it('should return __empty__ for empty arrays even without field config', () => { - const variable = { id: 'test_var' }; - const result = formatQuery([], variable); - expect(result).toBe('__empty__'); - }); - }); + describe('Empty arrays', () => { + it('should return __empty__ for empty arrays', () => { + const variable = { id: 'test_var', query: '{"field": "status"}' }; + const result = formatQuery([], variable); + expect(result).toBe('__empty__'); + }); - describe('Error handling and robustness', () => { - it('should not throw when variable.query is undefined', () => { - expect(() => { - formatQuery(['test'], { id: 'test_var' }); - }).not.toThrow(); + it('should return __empty__ for empty arrays even without field config', () => { + const variable = { id: 'test_var' }; + const result = formatQuery([], variable); + expect(result).toBe('__empty__'); + }); }); - it('should not throw when variable.query is malformed JSON', () => { - expect(() => { - formatQuery(['test'], { id: 'test_var', query: '{invalid json}' }); - }).not.toThrow(); - }); + describe('Error handling and robustness', () => { + it('should not throw when variable.query is undefined', () => { + expect(() => { + formatQuery(['test'], { id: 'test_var' }); + }).not.toThrow(); + }); - it('should not throw when variable.query is empty string', () => { - expect(() => { - formatQuery(['test'], { id: 'test_var', query: '' }); - }).not.toThrow(); - }); + it('should not throw when variable.query is malformed JSON', () => { + expect(() => { + formatQuery(['test'], { id: 'test_var', query: '{invalid json}' }); + }).not.toThrow(); + }); - it('should handle non-string, non-array values', () => { - const result = formatQuery(123 as any, { id: 'test_var' }); - expect(result).toBe('123'); - }); + it('should not throw when variable.query is empty string', () => { + expect(() => { + formatQuery(['test'], { id: 'test_var', query: '' }); + }).not.toThrow(); + }); - it('should handle boolean values', () => { - const result = formatQuery(true as any, { id: 'test_var' }); - expect(result).toBe('true'); - }); - }); + it('should handle non-string, non-array values', () => { + const result = formatQuery(123 as any, { id: 'test_var' }); + expect(result).toBe('123'); + }); - describe('Real-world scenarios', () => { - it('should handle variables from template variable queries', () => { - // Simulates a properly configured template variable - const templateVariable = { - id: 'log_level', - query: '{"field": "level"}' - }; - const result = formatQuery(['ERROR', 'WARN', 'INFO'], templateVariable); - expect(result).toBe('"ERROR" OR level:"WARN" OR level:"INFO"'); + it('should handle boolean values', () => { + const result = formatQuery(true as any, { id: 'test_var' }); + expect(result).toBe('true'); + }); }); - it('should handle variables without query configuration (legacy/simple variables)', () => { - // Simulates a simple template variable without field configuration - const simpleVariable = { - id: 'service_names' - }; - const result = formatQuery(['web-service', 'api-service'], simpleVariable); - expect(result).toBe('IN ["web\\-service" "api\\-service"]'); - }); + describe('Real-world scenarios', () => { + it('should handle variables from template variable queries', () => { + // Simulates a properly configured template variable + const templateVariable = { + id: 'log_level', + query: '{"field": "level"}' + }; + const result = formatQuery(['ERROR', 'WARN', 'INFO'], templateVariable); + expect(result).toBe('"ERROR" OR level:"WARN" OR level:"INFO"'); + }); - it('should handle variables with corrupted configuration', () => { - // Simulates a variable with corrupted/invalid configuration - const corruptedVariable = { - id: 'corrupted_var', - query: '{"field": undefined}' // Invalid JSON that might come from UI bugs - }; - const result = formatQuery(['value1', 'value2'], corruptedVariable); - expect(result).toBe('IN ["value1" "value2"]'); + it('should handle variables without query configuration (legacy/simple variables)', () => { + // Simulates a simple template variable without field configuration + const simpleVariable = { + id: 'service_names' + }; + const result = formatQuery(['web-service', 'api-service'], simpleVariable); + expect(result).toBe('"web\\-service" OR "api\\-service"'); + }); + + it('should handle variables with corrupted configuration', () => { + // Simulates a variable with corrupted/invalid configuration + const corruptedVariable = { + id: 'corrupted_var', + query: '{"field": undefined}' // Invalid JSON that might come from UI bugs + }; + const result = formatQuery(['value1', 'value2'], corruptedVariable); + expect(result).toBe('"value1" OR "value2"'); + }); }); }); - }); describe('quick filters', () => { const addFilterToQuery = ( @@ -186,7 +203,7 @@ describe('BaseQuickwitDataSource', () => { negate = false ) => { return (BaseQuickwitDataSource.prototype as any).addFilterToQuery.call( - { fieldTypes }, + Object.assign(Object.create(BaseQuickwitDataSource.prototype), { fieldTypes }), query, key, value, @@ -205,6 +222,9 @@ describe('BaseQuickwitDataSource', () => { ) as string; }; + const datasourceContext = (overrides: Record) => + Object.assign(Object.create(BaseQuickwitDataSource.prototype), overrides); + it('adds text filters with whitespace as phrase filters', () => { const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; @@ -261,6 +281,79 @@ describe('BaseQuickwitDataSource', () => { expect(updatedQuery.filters?.[0].filter.operator).toBe('term'); }); + it('reuses trailing empty filters without mutating the original query', () => { + const query = { + refId: 'A', + query: '', + metrics: [], + bucketAggs: [], + filters: [{ id: 'empty', filter: { key: '', operator: '=', value: '' } }], + } as any; + + const updatedQuery = addFilterToQuery( + { service_name: 'text' }, + query, + 'service_name', + 'frontend' + ); + + expect(updatedQuery).not.toBe(query); + expect(updatedQuery.filters?.[0]).toEqual({ + id: 'empty', + hide: false, + filter: { key: 'service_name', operator: 'term', value: 'frontend' }, + }); + expect(query.filters[0].filter).toEqual({ key: '', operator: '=', value: '' }); + }); + + it('adds quick filters when filters are undefined', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [] } as any; + + const updatedQuery = addFilterToQuery( + { service_name: 'text' }, + query, + 'service_name', + 'frontend' + ); + + expect(updatedQuery.filters).toEqual([ + expect.objectContaining({ + hide: false, + filter: { key: 'service_name', operator: 'term', value: 'frontend' }, + }), + ]); + }); + + it('keeps JSON array text filters as equality filters', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; + + const updatedQuery = addFilterToQuery( + { 'attributes.tags': 'text' }, + query, + 'attributes.tags', + '["paperclip"]' + ); + + expect(updatedQuery.filters?.[0].filter).toEqual({ + key: 'attributes.tags', + operator: '=', + value: '["paperclip"]', + }); + }); + + it('renders JSON array text filters with Quickwit array syntax', () => { + const result = renderAdHocFilters( + { 'attributes.tags': 'text' }, + [{ + key: 'attributes.tags', + operator: '=', + value: '["paperclip","stapler"]', + }] + ); + + expect(result).toBe('attributes.tags:IN ["paperclip" "stapler"]'); + }); + it('keeps punctuated text filters as phrase filters', () => { const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; @@ -301,15 +394,98 @@ describe('BaseQuickwitDataSource', () => { expect(result).toBe('attributes.grpc_message:error\\:foo'); }); + it('toggles off matching quick filters by operator', () => { + const query = { + refId: 'A', + query: '', + metrics: [], + bucketAggs: [], + filters: [{ id: 'existing', filter: { key: 'service_name', operator: 'term', value: 'frontend' } }], + } as any; + + const updatedQuery = (BaseQuickwitDataSource.prototype as any).toggleQueryFilter.call( + datasourceContext({ fieldTypes: { service_name: 'text' } }), + query, + { type: 'FILTER_FOR', options: { key: 'service_name', value: 'frontend' } } + ); + + expect(updatedQuery.filters).toEqual([{ id: expect.any(String), filter: { key: '', operator: '=', value: '' } }]); + }); + + it('replaces opposite quick filters instead of adding duplicates', () => { + const query = { + refId: 'A', + query: '', + metrics: [], + bucketAggs: [], + filters: [{ id: 'existing', filter: { key: 'service_name', operator: 'not term', value: 'frontend' } }], + } as any; + + const updatedQuery = (BaseQuickwitDataSource.prototype as any).toggleQueryFilter.call( + datasourceContext({ fieldTypes: { service_name: 'text' } }), + query, + { type: 'FILTER_FOR', options: { key: 'service_name', value: 'frontend' } } + ); + + expect(updatedQuery.filters).toEqual([ + { id: 'existing', hide: false, filter: { key: 'service_name', operator: 'term', value: 'frontend' } }, + ]); + }); + + it('can check quick filters by operator when Grafana provides filter direction', () => { + const query = { + refId: 'A', + query: '', + metrics: [], + bucketAggs: [], + filters: [{ id: 'existing', filter: { key: 'service_name', operator: 'not term', value: 'frontend' } }], + } as any; + const datasource = datasourceContext({ fieldTypes: { service_name: 'text' } }); + + expect((BaseQuickwitDataSource.prototype as any).queryHasFilter.call( + datasource, + query, + { key: 'service_name', value: 'frontend', type: 'FILTER_FOR' } + )).toBe(false); + expect((BaseQuickwitDataSource.prototype as any).queryHasFilter.call( + datasource, + query, + { key: 'service_name', value: 'frontend', type: 'FILTER_OUT' } + )).toBe(true); + }); + it('applies prior filters when loading tag values', async () => { const getTerms = jest.fn(() => from([[]])); await (BaseQuickwitDataSource.prototype as any).getTagValues.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteLimit: 1000, + filterAutocompleteUseFilterChains: true, + getTerms, + }), { + key: 'status', + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(getTerms).toHaveBeenCalledWith( + { field: 'status', query: 'service:"frontend"', size: 1000 }, + undefined + ); + }); + + it('uses the datasource autocomplete limit when loading tag values', async () => { + const getTerms = jest.fn(() => from([[]])); + + await (BaseQuickwitDataSource.prototype as any).getTagValues.call( + datasourceContext({ fieldTypes: {}, - addAdHocFilters: BaseQuickwitDataSource.prototype.addAdHocFilters, + filterAutocompleteLimit: 250, + filterAutocompleteUseFilterChains: true, getTerms, - }, + }), { key: 'status', filters: [{ key: 'service', operator: '=', value: 'frontend' }], @@ -317,9 +493,216 @@ describe('BaseQuickwitDataSource', () => { ); expect(getTerms).toHaveBeenCalledWith( - { field: 'status', query: 'service:"frontend"' }, + { field: 'status', query: 'service:"frontend"', size: 250 }, undefined ); }); + + it('can disable filter chains for tag values', async () => { + const getTerms = jest.fn(() => from([[]])); + + await (BaseQuickwitDataSource.prototype as any).getTagValues.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteLimit: 1000, + filterAutocompleteUseFilterChains: false, + getTerms, + }), + { + key: 'status', + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(getTerms).toHaveBeenCalledWith( + { field: 'status', query: '', size: 1000 }, + undefined + ); + }); + + it('derives chained tag keys from fields present in matching documents', async () => { + const getFields = jest.fn(() => + from([[ + { text: 'service', type: 'string' }, + { text: 'status', type: 'string' }, + { text: 'missing', type: 'string' }, + ]]) + ); + const query = jest.fn(() => + from([{ + data: [ + { + fields: [ + { name: 'service' }, + { name: 'status' }, + { name: 'sort' }, + ], + }, + ], + }]) + ); + + const result = await (BaseQuickwitDataSource.prototype as any).getTagKeys.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteLimit: 5, + filterAutocompleteUseFilterChains: true, + getFields, + query, + }), + { + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(result).toEqual([ + { text: 'service', type: 'string' }, + { text: 'status', type: 'string' }, + ]); + expect(query).toHaveBeenCalledWith( + expect.objectContaining({ + requestId: expect.stringMatching(/^getFilterKeys-/), + targets: [ + expect.objectContaining({ + query: 'service:"frontend"', + metrics: [{ id: 'filterKeys', type: 'raw_data', settings: { size: '5' } }], + }), + ], + }) + ); + }); + + it('derives full chained tag keys from every matching document', async () => { + const getFields = jest.fn(() => + from([[ + { text: 'service', type: 'string' }, + { text: 'status', type: 'string' }, + { text: 'rare.field', type: 'string' }, + { text: 'missing', type: 'string' }, + ]]) + ); + const postResource = jest.fn() + .mockResolvedValueOnce({ + num_hits: 2, + hits: [{ service: 'frontend', status: 'ok' }], + }) + .mockResolvedValueOnce({ + num_hits: 2, + hits: [{ service: 'frontend', rare: { field: 'present' } }], + }); + + const result = await (BaseQuickwitDataSource.prototype as any).getTagKeys.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteChainMode: 'full', + getFields, + index: 'logs', + postResource, + }), + { + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(result).toEqual([ + { text: 'service', type: 'string' }, + { text: 'status', type: 'string' }, + { text: 'rare.field', type: 'string' }, + ]); + expect(postResource).toHaveBeenCalledTimes(2); + expect(postResource).toHaveBeenNthCalledWith( + 1, + 'indexes/logs/search', + expect.objectContaining({ + query: 'service:"frontend"', + max_hits: 1000, + start_offset: 0, + }), + expect.objectContaining({ requestId: expect.stringMatching(/^getFilterKeysFull-/) }) + ); + expect(postResource).toHaveBeenNthCalledWith( + 2, + 'indexes/logs/search', + expect.objectContaining({ + query: 'service:"frontend"', + max_hits: 1000, + start_offset: 1, + }), + expect.objectContaining({ requestId: expect.stringMatching(/^getFilterKeysFull-/) }) + ); + }); + + it('deduplicates tag keys that have multiple field capabilities', async () => { + const getFields = jest.fn(() => + from([[ + { text: 'service_name', type: 'string' }, + { text: 'service_name', type: 'keyword' }, + { text: 'severity_text', type: 'string' }, + ]]) + ); + + const result = await (BaseQuickwitDataSource.prototype as any).getTagKeys.call( + datasourceContext({ + filterAutocompleteUseFilterChains: true, + getFields, + }), + {} + ); + + expect(result).toEqual([ + { text: 'service_name', type: 'string' }, + { text: 'severity_text', type: 'string' }, + ]); + }); + + it('can disable filter chains for tag keys', async () => { + const getFields = jest.fn(() => from([[{ text: 'service', type: 'string' }]])); + const query = jest.fn(); + + const result = await (BaseQuickwitDataSource.prototype as any).getTagKeys.call( + datasourceContext({ + filterAutocompleteUseFilterChains: false, + getFields, + query, + }), + { + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(result).toEqual([{ text: 'service', type: 'string' }]); + expect(query).not.toHaveBeenCalled(); + }); + }); + + describe('filter autocomplete limit', () => { + it('defaults invalid and empty values to 1000', () => { + expect(parseFilterAutocompleteLimit(undefined)).toBe(1000); + expect(parseFilterAutocompleteLimit('')).toBe(1000); + expect(parseFilterAutocompleteLimit('invalid')).toBe(1000); + expect(parseFilterAutocompleteLimit('-1')).toBe(1000); + }); + + it('accepts positive values and zero', () => { + expect(parseFilterAutocompleteLimit('250')).toBe(250); + expect(parseFilterAutocompleteLimit('0')).toBe(0); + }); + }); + + describe('filter autocomplete chain mode', () => { + it('defaults missing and invalid modes to sample', () => { + expect(parseFilterAutocompleteChainMode(undefined)).toBe('sample'); + expect(parseFilterAutocompleteChainMode('invalid')).toBe('sample'); + }); + + it('accepts supported modes', () => { + expect(parseFilterAutocompleteChainMode('none')).toBe('none'); + expect(parseFilterAutocompleteChainMode('sample')).toBe('sample'); + expect(parseFilterAutocompleteChainMode('full')).toBe('full'); + }); + + it('maps legacy disabled filter chains to none', () => { + expect(parseFilterAutocompleteChainMode(undefined, false)).toBe('none'); + }); }); }); diff --git a/src/datasource/base.ts b/src/datasource/base.ts index f91446d..bce19df 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -21,7 +21,7 @@ import { DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; -import { QuickwitOptions } from 'quickwit'; +import { FilterAutocompleteChainMode, QuickwitOptions } from 'quickwit'; import { getDataQuery } from 'QueryBuilder/elastic'; import { metricAggregationConfig } from 'components/QueryEditor/MetricAggregationsEditor/utils'; @@ -41,7 +41,34 @@ import { isSet } from '@/utils'; export type BaseQuickwitDataSourceConstructor = GConstructor -const getQueryUid = uidMaker("query") +const getQueryUid = uidMaker('query'); +export const DEFAULT_FILTER_AUTOCOMPLETE_LIMIT = 1000; +const DEFAULT_FILTER_AUTOCOMPLETE_CHAIN_MODE: FilterAutocompleteChainMode = 'sample'; +const FULL_FILTER_CHAIN_PAGE_SIZE = 1000; + +export function parseFilterAutocompleteLimit(value: unknown): number { + if (value === undefined || value === null || value === '') { + return DEFAULT_FILTER_AUTOCOMPLETE_LIMIT; + } + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 0) { + return DEFAULT_FILTER_AUTOCOMPLETE_LIMIT; + } + return Math.floor(parsed); +} + +export function parseFilterAutocompleteChainMode( + value: unknown, + legacyUseFilterChains?: boolean +): FilterAutocompleteChainMode { + if (value === 'none' || value === 'sample' || value === 'full') { + return value; + } + if (legacyUseFilterChains === false) { + return 'none'; + } + return DEFAULT_FILTER_AUTOCOMPLETE_CHAIN_MODE; +} type FieldCapsSpec = { aggregatable?: boolean, @@ -50,6 +77,11 @@ type FieldCapsSpec = { range?: TimeRange } +type QuickwitSearchResponse = { + num_hits?: number, + hits?: Array>, +} + export class BaseQuickwitDataSource extends DataSourceWithBackend implements @@ -63,6 +95,8 @@ export class BaseQuickwitDataSource queryEditorConfig?: { defaults?: DefaultsConfigOverrides }; + filterAutocompleteLimit: number; + filterAutocompleteChainMode: FilterAutocompleteChainMode; languageProvider: ElasticsearchLanguageProvider; // Populated lazily by getFields(). Used by modifyQuery to pick an operator // that matches text field semantics. @@ -85,6 +119,11 @@ export class BaseQuickwitDataSource this.logLevelField = settingsData.logLevelField || ''; this.dataLinks = settingsData.dataLinks || []; this.queryEditorConfig = settingsData.queryEditorConfig || {}; + this.filterAutocompleteLimit = parseFilterAutocompleteLimit(settingsData.filterAutocompleteLimit); + this.filterAutocompleteChainMode = parseFilterAutocompleteChainMode( + settingsData.filterAutocompleteChainMode, + settingsData.filterAutocompleteUseFilterChains + ); this.languageProvider = new ElasticsearchLanguageProvider(this); this.annotations = {}; @@ -154,66 +193,111 @@ export class BaseQuickwitDataSource const key = filter.options.key; const rawValue = String(filter.options.value ?? ''); const negate = filter.type === 'FILTER_OUT'; - - // If the same (key, value) filter is already present, toggle it off. - const existingIdx = (query.filters ?? []).findIndex( - (f) => f.filter.key === key && f.filter.value === rawValue + const operator = this.getFilterOperator(key, rawValue, negate); + const oppositeOperator = this.getFilterOperator(key, rawValue, !negate); + const emptyFilter = () => [{ id: newFilterId(), filter: { key: '', operator: '=', value: '' } }]; + const filters = query.filters ?? []; + + // If the same filter is already present, toggle it off. + const existingIdx = filters.findIndex( + (f) => !f.hide && f.filter.key === key && f.filter.value === rawValue && f.filter.operator === operator ); if (existingIdx !== -1) { - const next = [...(query.filters ?? [])]; + const next = [...filters]; next.splice(existingIdx, 1); - return { ...query, filters: next.length ? next : [{ id: newFilterId(), filter: { key: '', operator: '=', value: '' } }] }; + return { ...query, filters: next.length ? next : emptyFilter() }; } + + // If the opposite filter is present, replace it. + const oppositeIdx = filters.findIndex( + (f) => !f.hide && f.filter.key === key && f.filter.value === rawValue && f.filter.operator === oppositeOperator + ); + if (oppositeIdx !== -1) { + return { + ...query, + filters: filters.map((existing, index) => + index === oppositeIdx + ? { ...existing, hide: false, filter: { key, operator, value: rawValue } } + : existing + ), + }; + } + return this.addFilterToQuery(query, key, rawValue, negate); } - queryHasFilter(query: ElasticsearchQuery, filter: { key: string; value: string }): boolean { + queryHasFilter(query: ElasticsearchQuery, filter: { key: string; value: string; operator?: string; type?: string }): boolean { + const expectedOperator = filter.operator || ( + filter.type === 'FILTER_FOR' || filter.type === 'FILTER_OUT' + ? this.getFilterOperator(filter.key, String(filter.value ?? ''), filter.type === 'FILTER_OUT') + : undefined + ); return (query.filters ?? []).some( - (f) => f.filter.key === filter.key && f.filter.value === filter.value && !f.hide + (f) => + f.filter.key === filter.key && + f.filter.value === filter.value && + !f.hide && + (expectedOperator === undefined || f.filter.operator === expectedOperator) ); } - private addFilterToQuery(query: ElasticsearchQuery, key: string, rawValue: string, negate: boolean): ElasticsearchQuery { + private getFilterOperator(key: string, value: string, negate: boolean): string { const fieldType = this.fieldTypes[key]; const isText = fieldType === 'text'; - const value = rawValue; const useTermOperator = isText && value !== '' && isSimpleToken(value); - const operator = useTermOperator + return useTermOperator ? (negate ? 'not term' : 'term') : (negate ? '!=' : '='); + } + + private addFilterToQuery(query: ElasticsearchQuery, key: string, rawValue: string, negate: boolean): ElasticsearchQuery { + const operator = this.getFilterOperator(key, rawValue, negate); + const filters = query.filters ?? []; + const nextFilter = { key, operator, value: rawValue }; // If the user hasn't populated any filter yet, reuse the trailing empty one. - const len = query.filters?.length ?? 0; + const len = filters.length; if (len > 0) { - const last = query.filters![len - 1]; + const last = filters[len - 1]; if (!isSet(last.filter.key) && !isSet(last.filter.value)) { - last.filter.key = key; - last.filter.operator = operator; - last.filter.value = value; - return query; + return { + ...query, + filters: filters.map((filter, index) => + index === len - 1 ? { ...filter, hide: false, filter: nextFilter } : filter + ), + }; } } - query.filters?.push({ - id: newFilterId(), - hide: false, - filter: { key, operator, value }, - }); - return { ...query }; + return { + ...query, + filters: [ + ...filters, + { + id: newFilterId(), + hide: false, + filter: nextFilter, + }, + ], + }; } getDataQueryRequest(queryDef: TermsQuery, range: TimeRange, requestId?: string) { let dataQuery = getDataQuery(queryDef, 'getTerms'); + return this.getRequestForQuery(dataQuery, range, requestId); + } + + private getRequestForQuery(query: ElasticsearchQuery, range: TimeRange, requestId?: string) { const request: DataQueryRequest = { app: CoreApp.Unknown, requestId: requestId || getQueryUid.next(), interval: '', intervalMs: 0, range, - targets:[dataQuery], - timezone:'browser', - scopedVars:{}, + targets: [query], + timezone: 'browser', + scopedVars: {}, startTime: Date.now(), } return request @@ -294,17 +378,135 @@ export class BaseQuickwitDataSource * Get tag keys for adhoc filters */ getTagKeys(options: any = {}) { - const fields = this.getFields({aggregatable:true, range: options.timeRange}) - return lastValueFrom(fields, {defaultValue:[]}); + const fieldSpec = this.getTagKeyFieldSpec(options); + const filters = this.getFilterChain(options.filters); + if (!filters?.length) { + return lastValueFrom(this.getFields(fieldSpec), { defaultValue: [] }).then((fields) => this.dedupeTagKeys(fields)); + } + if (this.getFilterAutocompleteChainMode() === 'full') { + return this.getFullFilteredTagKeys(filters, fieldSpec); + } + return this.getSampledFilteredTagKeys(filters, fieldSpec); } /** * Get tag values for adhoc filters */ getTagValues(options: any) { - const query = this.addAdHocFilters('', options.filters); - const terms = this.getTerms({ field: options.key, query }, options.timeRange) - return lastValueFrom(terms, {defaultValue:[]}); + const query = this.addAdHocFilters('', this.getFilterChain(options.filters)); + const terms = this.getTerms({ field: options.key, query, size: this.filterAutocompleteLimit }, options.timeRange); + return lastValueFrom(terms, { defaultValue: [] }); + } + + private getFilterChain(filters?: AdHocVariableFilter[]) { + return this.getFilterAutocompleteChainMode() === 'none' ? undefined : filters; + } + + private getFilterAutocompleteChainMode() { + return parseFilterAutocompleteChainMode( + this.filterAutocompleteChainMode, + (this as { filterAutocompleteUseFilterChains?: boolean }).filterAutocompleteUseFilterChains + ); + } + + private getTagKeyFieldSpec(options: any = {}): FieldCapsSpec { + const spec: FieldCapsSpec = { range: options.timeRange }; + if (options.aggregatable !== undefined) { + spec.aggregatable = options.aggregatable; + } else if (options.searchable === undefined) { + spec.aggregatable = true; + } + if (options.searchable !== undefined) { + spec.searchable = options.searchable; + } + if (options.type !== undefined) { + spec.type = options.type; + } + return spec; + } + + private dedupeTagKeys(fields: MetricFindValue[]) { + const seen = new Set(); + return fields.filter((field) => { + const name = String(field.text); + if (seen.has(name)) { + return false; + } + seen.add(name); + return true; + }); + } + + private async getSampledFilteredTagKeys(filters: AdHocVariableFilter[], fieldSpec: FieldCapsSpec) { + const query = this.addAdHocFilters('', filters); + // Field-capabilities are schema-wide. To approximate chained field-key + // suggestions, sample matching documents and keep fields present in that + // sample. A limit of 0 means "no terms limit" elsewhere, but raw-document + // sampling still needs a finite size. + const sampleLimit = this.filterAutocompleteLimit > 0 ? this.filterAutocompleteLimit : DEFAULT_FILTER_AUTOCOMPLETE_LIMIT; + const target: ElasticsearchQuery = { + refId: 'filterKeys', + query, + metrics: [{ id: 'filterKeys', type: 'raw_data', settings: { size: sampleLimit.toString() } }], + bucketAggs: [], + filters: [], + }; + const range = fieldSpec.range || getDefaultTimeRange(); + const [fields, response] = await Promise.all([ + lastValueFrom(this.getFields(fieldSpec), { defaultValue: [] }), + lastValueFrom(this.query(this.getRequestForQuery(target, range, `getFilterKeys-${getQueryUid.next()}`)), { + defaultValue: { data: [] }, + }), + ]); + const presentFields = new Set(); + response.data?.forEach((frame: DataFrame) => { + frame.fields?.forEach((field: { name?: string }) => { + if (field.name && field.name !== 'sort') { + presentFields.add(field.name); + } + }); + }); + return this.dedupeTagKeys(fields).filter((field) => presentFields.has(String(field.text))); + } + + private async getFullFilteredTagKeys(filters: AdHocVariableFilter[], fieldSpec: FieldCapsSpec) { + const query = this.addAdHocFilters('', filters); + const range = fieldSpec.range || getDefaultTimeRange(); + const [fields, presentFields] = await Promise.all([ + lastValueFrom(this.getFields(fieldSpec), { defaultValue: [] }), + this.getAllMatchingFieldNames(query, range), + ]); + return this.dedupeTagKeys(fields).filter((field) => presentFields.has(String(field.text))); + } + + private async getAllMatchingFieldNames(query: string, range: TimeRange) { + const presentFields = new Set(); + let offset = 0; + let totalHits: number | undefined; + + do { + const response = await this.postResource( + `indexes/${this.index}/search`, + { + query: query || '*', + max_hits: FULL_FILTER_CHAIN_PAGE_SIZE, + start_offset: offset, + start_timestamp: Math.floor(range.from.valueOf() / SECOND), + end_timestamp: Math.ceil(range.to.valueOf() / SECOND), + }, + { requestId: `getFilterKeysFull-${getQueryUid.next()}` } + ); + const hits = response.hits ?? []; + totalHits = response.num_hits ?? totalHits ?? hits.length; + + hits.forEach((hit) => collectFieldNames(hit, presentFields)); + offset += hits.length; + if (hits.length === 0) { + break; + } + } while (totalHits === undefined || offset < totalHits); + + return presentFields; } /** @@ -388,7 +590,9 @@ export class BaseQuickwitDataSource } interpolateLuceneQuery(queryString: string, scopedVars?: ScopedVars) { - return this.templateSrv.replace(queryString, scopedVars, formatQuery); + return this.templateSrv.replace(queryString, scopedVars, (value: string | string[], variable: any) => + formatQuery(value, variable, queryString) + ); } interpolateVariablesInQueries(queries: ElasticsearchQuery[], scopedVars: ScopedVars | {}, filters?: AdHocVariableFilter[]): ElasticsearchQuery[] { @@ -475,7 +679,50 @@ export class BaseQuickwitDataSource return finalQuery; } } -export function formatQuery(value: string | string[], variable: any): string { + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function collectFieldNames(value: unknown, fields: Set, prefix = '') { + if (Array.isArray(value)) { + value.forEach((item) => collectFieldNames(item, fields, prefix)); + return; + } + if (!isRecord(value)) { + if (prefix) { + fields.add(prefix); + } + return; + } + + Object.entries(value).forEach(([key, nested]) => { + const fieldName = prefix ? `${prefix}.${key}` : key; + collectFieldNames(nested, fields, fieldName); + }); +} + +function escapeRegExp(value: string) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function inferFieldNameFromQuery(queryString: string | undefined, variable: any) { + const names = [variable?.id, variable?.name].filter((name): name is string => typeof name === 'string' && name !== ''); + if (!queryString || names.length === 0) { + return undefined; + } + for (const name of names) { + const escapedName = escapeRegExp(name); + const pattern = new RegExp(`(?:^|[\\s(])([^\\s:()]+):(\\$\\{${escapedName}\\}|\\$${escapedName}|\\[\\[${escapedName}\\]\\])`); + const match = queryString.match(pattern); + if (match?.[1]) { + return match[1]; + } + } + return undefined; +} + +export function formatQuery(value: string | string[], variable: any, queryString?: string): string { if (typeof value === 'string') { return luceneEscape(value); } @@ -489,19 +736,17 @@ export function formatQuery(value: string | string[], variable: any): string { } catch (e) { fieldName = undefined; } + fieldName = typeof fieldName === 'string' ? fieldName : inferFieldNameFromQuery(queryString, variable); const quotedValues = value.map((val) => '"' + luceneEscape(val) + '"'); // Quickwit query language does not support fieldName:(value1 OR value2 OR....) // like lucene does. // When we know the fieldName, we can directly generate a query // fieldName:value1 OR fieldName:value2 OR ... - // But when we don't know the fieldName, the simplest is to generate a query - // with the IN operator. Unfortunately, IN operator does not work on JSON field. - // TODO: fix that by using doing a regex on queryString to find the fieldName. - // Note that variable.id gives the name of the template variable to interpolate, - // so if we have `fieldName:${variable.id}` in the queryString, we can isolate - // the fieldName. + // If the variable query does not carry a field, infer it from common + // `field:$var` query strings. Without a field, fall back to default-search + // clauses instead of a bare IN set, which is easy to misread as field-scoped. if (typeof fieldName !== 'string') { - return 'IN [' + quotedValues.join(' ') + ']'; + return quotedValues.join(' OR '); } return quotedValues.join(' OR ' + fieldName + ':'); } else { diff --git a/src/modifyQuery.test.ts b/src/modifyQuery.test.ts index 85da34b..aa4772d 100644 --- a/src/modifyQuery.test.ts +++ b/src/modifyQuery.test.ts @@ -74,6 +74,60 @@ describe('addAddHocFilter', () => { expect(result).toBe('attributes.tags:IN ["foo bar" "baz qux"]'); }); + it('handles single-element numeric arrays', () => { + const result = addAddHocFilter('', { + key: 'attributes.codes', + operator: '=', + value: '[200]', + }); + expect(result).toBe('attributes.codes:200'); + }); + + it('handles multi-element numeric arrays with IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.codes', + operator: '=', + value: '[200,500]', + }); + expect(result).toBe('attributes.codes:IN [200 500]'); + }); + + it('keeps negative numeric array values unquoted', () => { + const result = addAddHocFilter('', { + key: 'attributes.deltas', + operator: '=', + value: '[-1,2]', + }); + expect(result).toBe('attributes.deltas:IN [-1 2]'); + }); + + it('handles boolean arrays with IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.flags', + operator: '=', + value: '[true,false]', + }); + expect(result).toBe('attributes.flags:IN [true false]'); + }); + + it('handles mixed scalar arrays with IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.values', + operator: '=', + value: '["paperclip",200,true]', + }); + expect(result).toBe('attributes.values:IN ["paperclip" 200 true]'); + }); + + it('negates numeric arrays with IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.codes', + operator: '!=', + value: '[200,500]', + }); + expect(result).toBe('-attributes.codes:IN [200 500]'); + }); + it('handles array values containing double quotes', () => { const result = addAddHocFilter('', { key: 'attributes.tags', @@ -130,6 +184,42 @@ describe('addAddHocFilter', () => { expect(result).toBe('status:200 AND attributes.controller:"BlogController"'); }); + it('renders numeric equality filters as unquoted literals', () => { + const result = addAddHocFilter('', { + key: 'attributes.status_code', + operator: '=', + value: '200', + }); + expect(result).toBe('attributes.status_code:200'); + }); + + it('keeps numeric zero as a valid filter value', () => { + const result = addAddHocFilter('', { + key: 'attributes.retry_count', + operator: '=', + value: 0 as any, + }); + expect(result).toBe('attributes.retry_count:0'); + }); + + it('keeps boolean false as a valid filter value', () => { + const result = addAddHocFilter('', { + key: 'attributes.cache_hit', + operator: '=', + value: false as any, + }); + expect(result).toBe('attributes.cache_hit:false'); + }); + + it('renders negated boolean equality filters as unquoted literals', () => { + const result = addAddHocFilter('', { + key: 'attributes.cache_hit', + operator: '!=', + value: false as any, + }); + expect(result).toBe('-attributes.cache_hit:false'); + }); + it('exists operator', () => { const result = addAddHocFilter('', { key: 'attributes.tags', diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index 3141735..5fed8b7 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -1,13 +1,19 @@ import { escapeFilter, escapeFilterValue, concatenate, LuceneQuery } from 'utils/lucene'; import { AdHocVariableFilter } from '@grafana/data'; -function tryParseJsonArray(value: string): string[] | null { - if (!value.startsWith('[')) { +type FilterArrayElement = string | number | boolean; + +function isFilterArrayElement(value: unknown): value is FilterArrayElement { + return ['string', 'number', 'boolean'].includes(typeof value); +} + +function tryParseJsonArray(value: string): FilterArrayElement[] | null { + if (!value.trimStart().startsWith('[')) { return null; } try { const parsed = JSON.parse(value); - if (Array.isArray(parsed) && parsed.every((el) => typeof el === 'string')) { + if (Array.isArray(parsed) && parsed.every(isFilterArrayElement)) { return parsed; } } catch { @@ -16,11 +22,26 @@ function tryParseJsonArray(value: string): string[] | null { return null; } +function formatArrayElement(value: FilterArrayElement) { + if (typeof value === 'string') { + return `"${escapeFilterValue(value)}"`; + } + return String(value); +} + +function hasFilterValue(value: unknown) { + return value !== undefined && value !== null && String(value) !== ''; +} + +function isLiteralValue(value: string) { + return /^-?(?:\d+(?:\.\d+)?|\.\d+)(?:e[+-]?\d+)?$/i.test(value) || value === 'true' || value === 'false'; +} + /** * Adds a label:"value" expression to the query. */ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): string { - const hasValidValue = ['exists', 'not exists'].includes(filter.operator) || !!filter.value + const hasValidValue = ['exists', 'not exists'].includes(filter.operator) || hasFilterValue(filter.value) if (!filter.key || !hasValidValue) { return query; } @@ -33,6 +54,8 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str const equalityFilters = ['=', '!=']; if (equalityFilters.includes(filter.operator)) { + const modifier = filter.operator === '=' ? '' : '-'; + const key = escapeFilter(filter.key); // Grafana stringifies array values (e.g. ["paperclip","stapler"]) before // passing them as filter values. Tantivy indexes array elements as // individual terms — there's no way to match on array length, order, or @@ -43,14 +66,15 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str if (arrayElements.length === 0) { return query; } - const modifier = filter.operator === '=' ? '' : '-'; - const key = escapeFilter(filter.key); if (arrayElements.length === 1) { - return concatenate(query, `${modifier}${key}:"${escapeFilterValue(arrayElements[0])}"`, 'AND'); + return concatenate(query, `${modifier}${key}:${formatArrayElement(arrayElements[0])}`, 'AND'); } - const terms = arrayElements.map((el) => `"${escapeFilterValue(el)}"`).join(' '); + const terms = arrayElements.map(formatArrayElement).join(' '); return concatenate(query, `${modifier}${key}:IN [${terms}]`, 'AND'); } + if (isLiteralValue(filter.value)) { + return concatenate(query, `${modifier}${key}:${filter.value}`, 'AND'); + } return LuceneQuery.parse(query).addFilter(filter.key, filter.value, filter.operator === '=' ? '' : '-').toString(); } /** diff --git a/src/quickwit.ts b/src/quickwit.ts index 2d1599f..8c38aa9 100644 --- a/src/quickwit.ts +++ b/src/quickwit.ts @@ -2,6 +2,8 @@ import { DataSourceJsonData } from "@grafana/data"; import { DataLinkConfig } from "./types"; import { DefaultsConfigOverrides } from "store/defaults/conf"; +export type FilterAutocompleteChainMode = 'none' | 'sample' | 'full'; + export interface QuickwitOptions extends DataSourceJsonData { timeField: string; interval?: string; @@ -9,6 +11,10 @@ export interface QuickwitOptions extends DataSourceJsonData { logLevelField?: string; dataLinks?: DataLinkConfig[]; index: string; + filterAutocompleteLimit?: string; + filterAutocompleteChainMode?: FilterAutocompleteChainMode; + // Backward compatibility for configs created before the mode selector. + filterAutocompleteUseFilterChains?: boolean; queryEditorConfig?: { defaults?: DefaultsConfigOverrides } From 53b3570c0fb7da533654c5d0e812eb290e6d0692 Mon Sep 17 00:00:00 2001 From: Patrik Cyvoct Date: Mon, 20 Apr 2026 09:20:19 +0200 Subject: [PATCH 14/14] Bump to version 0.6.0 Signed-off-by: Patrik Cyvoct --- CHANGELOG.md | 21 +++++++++++++++++++++ README.md | 41 +++++++++++++++++++++++------------------ package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63e3286..080b16d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 0.6.0 + +### What's Changed + +- React 19 compatibility for Grafana 13 +- Add quick filters with value autocomplete and clearer filter operations +- Add trace search and full trace rendering for OpenTelemetry trace indexes, with Grafana trace frames, service node graph, span events, exception stack traces, status/warning handling, service tags, peer service metadata, and stable per-service node colors +- Add trace-to-logs and log-to-trace correlation links between separate Quickwit logs and traces datasources +- Add datasource configuration fields for related logs/traces datasources +- Add configurable filter autocomplete chain mode (no chain, sampled chain, full chain) +- Add configurable filter autocomplete value limit (defaults to 1000, `0` for unlimited) +- Show useful default log messages for OTEL logs when no message field is configured +- Migrate e2e tests from Cypress to Playwright and add Quickwit datasource e2e test +- Add Grafana version matrix to e2e tests +- Fix Shift-Enter keymapping on latest Grafana versions +- Fix missing `AND` when adding ad hoc filters +- Document Grafana 12.1+ plugin installation with `GF_PLUGINS_PREINSTALL_SYNC` +- Handle Grafana 13 overlays in e2e tests +- Bump Go and Node dependencies to fix high vulnerabilities +- Bump `grafana/plugin-actions/is-compatible` GitHub action + ## 0.5.0 ### What's Changed diff --git a/README.md b/README.md index b5207fb..e882f85 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,17 @@ The Quickwit data source plugin allows you to query and visualize Quickwit data from within Grafana. -## 🎉 What's New in v0.5.0 - -- **Grafana 11.x Support** -- **Fixed Adhoc Filters**: Improved adhoc filters feature for dynamic query building -- **Enhanced Stability**: Various bug fixes and improvements +## 🎉 What's New in v0.6.0 + +- **Grafana 12.1+ and 13 Support** (React 19 compatibility) +- **Trace support** for OpenTelemetry trace indexes, with service node graph, span events, exception stacks, and trace/logs correlation links +- **Quick filters** with value autocomplete and clearer filter operations +- **Configurable autocomplete** chain mode and value limit +- **Better OTEL log display** when no message field is configured +- **Playwright e2e tests** replacing Cypress, with a Grafana version matrix +- **Fixed Shift-Enter keymapping** on latest Grafana versions +- **Fixed ad hoc filters** when appending filters to existing queries +- **Security updates**: Go and Node dependency bumps It is available for installation directly from the [Grafana catalog](https://grafana.com/grafana/plugins/quickwit-quickwit-datasource/) until version 0.4.5 @@ -17,40 +23,39 @@ or you can download the latest version and follow the ## Version compatibility -We recommend Grafana v10.X or v11.X. +We recommend Grafana v12.1+ or v13. Quickwit 0.7 is compatible with 0.3.x versions only. -Quickwit 0.8 is compatible with 0.4.x and 0.5.x versions. +Quickwit 0.8 is compatible with 0.4.x, 0.5.x and 0.6.x versions. -- **v0.5.x** (Latest): Grafana 11.x with improved adhoc filters -- **v0.4.x**: Grafana 10.x +- **v0.6.x** (Latest): Grafana 12.1+ and 13 (React 19) +- **v0.5.x**: Grafana 11.x +- **v0.4.x**: Grafana 10.x - **v0.3.x**: Grafana 9.x / Quickwit 0.7 ## Installation -You can either download the plugin manually and unzip it into the plugin directory or use the env variable `GF_INSTALL_PLUGINS` to install it. - -### 0.5.0 (Latest) for Quickwit 0.8 + Grafana 12.1 +You can either download the plugin manually and unzip it into the plugin directory, or use a Grafana env variable to install it. Note that `GF_INSTALL_PLUGINS` is **deprecated since Grafana 12.1** — use `GF_PLUGINS_PREINSTALL_SYNC` instead on recent versions. -`GF_INSTALL_PLUGINS` has been deprecated since 12.1. `GF_PLUGINS_PREINSTALL_SYNC` must be used instead +### 0.6.0 (Latest) for Quickwit 0.8 + Grafana 12.1+ / 13 -Run `grafana` container with the env variable: +Run `grafana` container with the env variable (format: `@@`): ```bash -docker run -p 3000:3000 -e GF_PLUGINS_PREINSTALL_SYNC="quickwit-quickwit-datasource@0.5.0@https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.5.0/quickwit-quickwit-datasource-0.5.0.zip" grafana/grafana run +docker run -p 3000:3000 -e GF_PLUGINS_PREINSTALL_SYNC="quickwit-quickwit-datasource@0.6.0@https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.6.0/quickwit-quickwit-datasource-0.6.0.zip" grafana/grafana run ``` Or download the plugin manually and start Grafana ```bash -wget https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.5.0/quickwit-quickwit-datasource-0.5.0.zip +wget https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.6.0/quickwit-quickwit-datasource-0.6.0.zip mkdir -p plugins -unzip quickwit-quickwit-datasource-0.5.0.zip -d plugins/quickwit-quickwit-datasource-0.5.0 +unzip quickwit-quickwit-datasource-0.6.0.zip -d plugins/quickwit-quickwit-datasource-0.6.0 docker run -p 3000:3000 -e GF_PATHS_PLUGINS=/data/plugins -v ${PWD}/plugins:/data/plugins grafana/grafana run ``` -### 0.5.0 (Latest) for Quickwit 0.8 + Grafana 11 +### 0.5.0 for Quickwit 0.8 + Grafana 11 Run `grafana` container with the env variable: diff --git a/package-lock.json b/package-lock.json index 833a97e..fb1773c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "quickwit-datasource", - "version": "0.5.0", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "quickwit-datasource", - "version": "0.5.0", + "version": "0.6.0", "license": "AGPL-3.0", "dependencies": { "@codemirror/commands": "^6.8.1", diff --git a/package.json b/package.json index 3ab9ce2..a59970e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "quickwit-datasource", - "version": "0.5.0", + "version": "0.6.0", "description": "Quickwit datasource", "scripts": { "build": "webpack -c ./.config/webpack/webpack.config.ts --env production",