diff --git a/CHANGELOG.md b/CHANGELOG.md index 63e3286..080b16d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 0.6.0 + +### What's Changed + +- React 19 compatibility for Grafana 13 +- Add quick filters with value autocomplete and clearer filter operations +- Add trace search and full trace rendering for OpenTelemetry trace indexes, with Grafana trace frames, service node graph, span events, exception stack traces, status/warning handling, service tags, peer service metadata, and stable per-service node colors +- Add trace-to-logs and log-to-trace correlation links between separate Quickwit logs and traces datasources +- Add datasource configuration fields for related logs/traces datasources +- Add configurable filter autocomplete chain mode (no chain, sampled chain, full chain) +- Add configurable filter autocomplete value limit (defaults to 1000, `0` for unlimited) +- Show useful default log messages for OTEL logs when no message field is configured +- Migrate e2e tests from Cypress to Playwright and add Quickwit datasource e2e test +- Add Grafana version matrix to e2e tests +- Fix Shift-Enter keymapping on latest Grafana versions +- Fix missing `AND` when adding ad hoc filters +- Document Grafana 12.1+ plugin installation with `GF_PLUGINS_PREINSTALL_SYNC` +- Handle Grafana 13 overlays in e2e tests +- Bump Go and Node dependencies to fix high vulnerabilities +- Bump `grafana/plugin-actions/is-compatible` GitHub action + ## 0.5.0 ### What's Changed diff --git a/README.md b/README.md index 901133a..e882f85 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,17 @@ The Quickwit data source plugin allows you to query and visualize Quickwit data from within Grafana. -## 🎉 What's New in v0.5.0 - -- **Grafana 11.x Support** -- **Fixed Adhoc Filters**: Improved adhoc filters feature for dynamic query building -- **Enhanced Stability**: Various bug fixes and improvements +## 🎉 What's New in v0.6.0 + +- **Grafana 12.1+ and 13 Support** (React 19 compatibility) +- **Trace support** for OpenTelemetry trace indexes, with service node graph, span events, exception stacks, and trace/logs correlation links +- **Quick filters** with value autocomplete and clearer filter operations +- **Configurable autocomplete** chain mode and value limit +- **Better OTEL log display** when no message field is configured +- **Playwright e2e tests** replacing Cypress, with a Grafana version matrix +- **Fixed Shift-Enter keymapping** on latest Grafana versions +- **Fixed ad hoc filters** when appending filters to existing queries +- **Security updates**: Go and Node dependency bumps It is available for installation directly from the [Grafana catalog](https://grafana.com/grafana/plugins/quickwit-quickwit-datasource/) until version 0.4.5 @@ -17,21 +23,39 @@ or you can download the latest version and follow the ## Version compatibility -We recommend Grafana v10.X or v11.X. +We recommend Grafana v12.1+ or v13. Quickwit 0.7 is compatible with 0.3.x versions only. -Quickwit 0.8 is compatible with 0.4.x and 0.5.x versions. +Quickwit 0.8 is compatible with 0.4.x, 0.5.x and 0.6.x versions. -- **v0.5.x** (Latest): Grafana 11.x with improved adhoc filters -- **v0.4.x**: Grafana 10.x +- **v0.6.x** (Latest): Grafana 12.1+ and 13 (React 19) +- **v0.5.x**: Grafana 11.x +- **v0.4.x**: Grafana 10.x - **v0.3.x**: Grafana 9.x / Quickwit 0.7 ## Installation -You can either download the plugin manually and unzip it into the plugin directory or use the env variable `GF_INSTALL_PLUGINS` to install it. +You can either download the plugin manually and unzip it into the plugin directory, or use a Grafana env variable to install it. Note that `GF_INSTALL_PLUGINS` is **deprecated since Grafana 12.1** — use `GF_PLUGINS_PREINSTALL_SYNC` instead on recent versions. + +### 0.6.0 (Latest) for Quickwit 0.8 + Grafana 12.1+ / 13 + +Run `grafana` container with the env variable (format: `@@`): + +```bash +docker run -p 3000:3000 -e GF_PLUGINS_PREINSTALL_SYNC="quickwit-quickwit-datasource@0.6.0@https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.6.0/quickwit-quickwit-datasource-0.6.0.zip" grafana/grafana run +``` + +Or download the plugin manually and start Grafana + +```bash +wget https://github.com/quickwit-oss/quickwit-datasource/releases/download/v0.6.0/quickwit-quickwit-datasource-0.6.0.zip +mkdir -p plugins +unzip quickwit-quickwit-datasource-0.6.0.zip -d plugins/quickwit-quickwit-datasource-0.6.0 +docker run -p 3000:3000 -e GF_PATHS_PLUGINS=/data/plugins -v ${PWD}/plugins:/data/plugins grafana/grafana run +``` -### 0.5.0 (Latest) for Quickwit 0.8 + Grafana 11 +### 0.5.0 for Quickwit 0.8 + Grafana 11 Run `grafana` container with the env variable: diff --git a/package-lock.json b/package-lock.json index 833a97e..fb1773c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "quickwit-datasource", - "version": "0.5.0", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "quickwit-datasource", - "version": "0.5.0", + "version": "0.6.0", "license": "AGPL-3.0", "dependencies": { "@codemirror/commands": "^6.8.1", diff --git a/package.json b/package.json index 3ab9ce2..a59970e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "quickwit-datasource", - "version": "0.5.0", + "version": "0.6.0", "description": "Quickwit datasource", "scripts": { "build": "webpack -c ./.config/webpack/webpack.config.ts --env production", diff --git a/src/QueryBuilder/elastic.test.ts b/src/QueryBuilder/elastic.test.ts new file mode 100644 index 0000000..28ebbed --- /dev/null +++ b/src/QueryBuilder/elastic.test.ts @@ -0,0 +1,25 @@ +import { getDataQuery } from './elastic'; + +describe('getDataQuery', () => { + it('uses the requested terms size for autocomplete queries', () => { + const query = getDataQuery({ field: 'status', size: 250 }, 'getTerms'); + + expect(query.bucketAggs?.[0].settings).toEqual( + expect.objectContaining({ + size: '250', + shard_size: '250', + }) + ); + }); + + it('keeps zero as no terms limit', () => { + const query = getDataQuery({ field: 'status', size: 0 }, 'getTerms'); + + expect(query.bucketAggs?.[0].settings).toEqual( + expect.objectContaining({ + size: '0', + shard_size: '0', + }) + ); + }); +}); diff --git a/src/QueryBuilder/elastic.ts b/src/QueryBuilder/elastic.ts index 907f66e..d6df917 100644 --- a/src/QueryBuilder/elastic.ts +++ b/src/QueryBuilder/elastic.ts @@ -58,7 +58,8 @@ export function getDataQuery(queryDef: TermsQuery, refId: string): Elasticsearch const bucketAggs: BucketAggregation[] = []; if (queryDef.field) { - bucketAggs.push(getTermsAgg(queryDef.field, 100, 100, orderBy, order)) + const size = queryDef.size ?? 100; + bucketAggs.push(getTermsAgg(queryDef.field, size, size, orderBy, order)) } return { diff --git a/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx b/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx index 8b473ae..f8be73f 100644 --- a/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx +++ b/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx @@ -22,6 +22,7 @@ describe('Terms Settings Editor', () => { query: '', bucketAggs: [termsAgg], metrics: [avg, derivative, topMetrics], + filters: [], }; renderWithESProvider(, { providerProps: { query } }); diff --git a/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx b/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx index bc52130..4db8980 100644 --- a/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx +++ b/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx @@ -13,6 +13,7 @@ const query: ElasticsearchQuery = { query: '', metrics: [{ id: '1', type: 'count' }], bucketAggs: [{ type: 'date_histogram', id: '2' }], + filters: [] }; describe('ElasticsearchQueryContext', () => { diff --git a/src/components/QueryEditor/ElasticsearchQueryContext.tsx b/src/components/QueryEditor/ElasticsearchQueryContext.tsx index 7595f75..6c8dddf 100644 --- a/src/components/QueryEditor/ElasticsearchQueryContext.tsx +++ b/src/components/QueryEditor/ElasticsearchQueryContext.tsx @@ -8,6 +8,7 @@ import { ElasticsearchQuery } from '@/types'; import { createReducer as createBucketAggsReducer } from './BucketAggregationsEditor/state/reducer'; import { reducer as metricsReducer } from './MetricAggregationsEditor/state/reducer'; +import { reducer as filtersReducer } from './FilterEditor/state/reducer'; import { aliasPatternReducer, queryReducer, initQuery, initExploreQuery } from './state'; import { getHook } from '@/utils/context'; import { Provider, useDispatch } from "react-redux"; @@ -64,10 +65,11 @@ export const ElasticsearchProvider = withStore(({ [onChange] ); - const reducer = combineReducers>({ + const reducer = combineReducers>({ query: queryReducer, alias: aliasPatternReducer, metrics: metricsReducer, + filters: filtersReducer, bucketAggs: createBucketAggsReducer(datasource.timeField), }); @@ -78,7 +80,7 @@ export const ElasticsearchProvider = withStore(({ reducer ); - const isUninitialized = !query.metrics || !query.bucketAggs || query.query === undefined; + const isUninitialized = !query.metrics || !query.filters || !query.bucketAggs || query.query === undefined; const [shouldRunInit, setShouldRunInit] = useState(isUninitialized); diff --git a/src/components/QueryEditor/FilterEditor/index.test.tsx b/src/components/QueryEditor/FilterEditor/index.test.tsx new file mode 100644 index 0000000..b648254 --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/index.test.tsx @@ -0,0 +1,50 @@ +import { QueryFilter } from '@/types'; + +import { getPreviousAdHocFilters } from './index'; + +describe('FilterEditor helpers', () => { + it('returns only complete filters before the current filter', () => { + const filters: QueryFilter[] = [ + { + id: 'first', + filter: { key: 'service', operator: '=', value: 'frontend' }, + }, + { + id: 'hidden', + hide: true, + filter: { key: 'cluster', operator: '=', value: 'prod' }, + }, + { + id: 'incomplete', + filter: { key: 'namespace', operator: '=', value: '' }, + }, + { + id: 'current', + filter: { key: 'attributes.grpc_message', operator: '=', value: '' }, + }, + { + id: 'later', + filter: { key: 'status', operator: '=', value: '500' }, + }, + ]; + + expect(getPreviousAdHocFilters(filters, 'current')).toEqual([ + { key: 'service', operator: '=', value: 'frontend' }, + ]); + }); + + it('does not include previous term filters with whitespace values', () => { + const filters: QueryFilter[] = [ + { + id: 'invalid-term', + filter: { key: 'message', operator: 'term', value: 'invalid token' }, + }, + { + id: 'current', + filter: { key: 'status', operator: '=', value: '' }, + }, + ]; + + expect(getPreviousAdHocFilters(filters, 'current')).toEqual([]); + }); +}); diff --git a/src/components/QueryEditor/FilterEditor/index.tsx b/src/components/QueryEditor/FilterEditor/index.tsx new file mode 100644 index 0000000..0c879dc --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/index.tsx @@ -0,0 +1,215 @@ +import React from 'react'; + +import { useDispatch } from '@/hooks/useStatelessReducer'; +import { IconButton } from '../../IconButton'; +import { useDatasource, useQuery, useRange } from '../ElasticsearchQueryContext'; +import { QueryEditorRow } from '../QueryEditorRow'; + +import { QueryFilter } from '@/types'; +import { Icon, InlineSegmentGroup, Segment, SegmentAsync, Tooltip } from '@grafana/ui'; +import { AdHocVariableFilter, MetricFindValue, SelectableValue } from '@grafana/data'; +import { + addFilter, + removeFilter, + toggleFilterVisibility, + changeFilterField, + changeFilterOperation, + changeFilterValue, +} from '@/components/QueryEditor/FilterEditor/state/actions'; +import { segmentStyles } from '@/components/QueryEditor/styles'; +import { newFilterId } from '@/utils/uid'; +import { categorizeFieldType, filterOperations, filterOperationsFor } from '@/queryDef'; +import { fuzzySearchSort, hasWhiteSpace, isSet } from '@/utils'; + +interface FilterEditorProps { + onSubmit: () => void; +} + +function filterErrors(filter: QueryFilter): string[] { + const errors: string[] = []; + + if (!isSet(filter.filter.key)) { + errors.push('Field is not set'); + } + + if (!isSet(filter.filter.operator)) { + errors.push('Operator is not set'); + } + + if (!['exists', 'not exists'].includes(filter.filter.operator) && !isSet(filter.filter.value)) { + errors.push('Value is not set'); + } + + if (['term', 'not term'].includes(filter.filter.operator) && filter.filter.value && hasWhiteSpace(filter.filter.value)) { + errors.push('Term cannot have whitespace in value'); + } + + return errors; +} + +function isFilterComplete(filter: QueryFilter): boolean { + return !filter.hide && filterErrors(filter).length === 0; +} + +export function getPreviousAdHocFilters(filters: QueryFilter[] | undefined, currentId: QueryFilter['id']): AdHocVariableFilter[] { + const currentIndex = filters?.findIndex((filter) => filter.id === currentId) ?? -1; + if (!filters || currentIndex <= 0) { + return []; + } + + return filters + .slice(0, currentIndex) + .filter(isFilterComplete) + .map((filter) => filter.filter); +} + +function toFuzzyOptions(values: MetricFindValue[], query?: string): Array> { + return fuzzySearchSort( + values.map((value) => String(value.text)), + (text) => text, + query + ).map((text) => ({ label: text, value: text })); +} + +export const FilterEditor = ({ onSubmit }: FilterEditorProps) => { + const dispatch = useDispatch(); + const { filters } = useQuery(); + + return ( + <> + {filters?.map((filter, index) => { + const errors = filterErrors(filter) + return ( + 0 ? ( + + Filter + + ): 'Filter'} + hidden={filter.hide} + onHideClick={() => { + dispatch(toggleFilterVisibility(filter.id)); + onSubmit(); + }} + onRemoveClick={() => { + dispatch(removeFilter(filter.id)); + onSubmit(); + }} + > + + + {index === 0 && dispatch(addFilter(newFilterId()))} + />} + + ) + })} + + ); +}; + +interface FilterEditorRowProps { + value: QueryFilter; + onSubmit: () => void; +} + +export const FilterEditorRow = ({ value, onSubmit }: FilterEditorRowProps) => { + const dispatch = useDispatch(); + const datasource = useDatasource(); + const range = useRange(); + const { filters } = useQuery(); + const previousFilters = getPreviousAdHocFilters(filters, value.id); + + const fieldCategory = categorizeFieldType(datasource.getFieldType?.(value.filter.key)); + const visibleOperations = filterOperationsFor(fieldCategory); + + const loadFields = async (query?: string): Promise>> => { + const values = await datasource.getTagKeys({ filters: previousFilters, timeRange: range }); + return toFuzzyOptions(values as MetricFindValue[], query); + }; + + const loadValues = async (query?: string): Promise>> => { + if (!isSet(value.filter.key) || !datasource.getTagValues) { + return []; + } + const values: MetricFindValue[] = await datasource.getTagValues({ + key: value.filter.key, + filters: previousFilters, + timeRange: range, + }); + return toFuzzyOptions(values, query); + }; + + return ( + <> + + { + const newKey = e.value ?? ''; + dispatch(changeFilterField({ id: value.id, field: newKey })); + // If the currently selected operator isn't valid for the new field, + // reset it to the first valid one so the UI stays consistent. + const newCategory = categorizeFieldType(datasource.getFieldType?.(newKey)); + const allowed = filterOperationsFor(newCategory); + if (!allowed.some((op) => op.value === value.filter.operator)) { + dispatch(changeFilterOperation({ id: value.id, op: allowed[0].value })); + } + if (['exists', 'not exists'].includes(value.filter.operator) || isSet(value.filter.value)) { + onSubmit(); + } + }} + placeholder="Select Field" + value={value.filter.key} + /> +
+ op.value === value.filter.operator)} + options={visibleOperations} + onChange={(e) => { + let op = e.value ?? filterOperations[0].value; + dispatch(changeFilterOperation({ id: value.id, op: op })); + if (['exists', 'not exists'].includes(op) || isSet(value.filter.value)) { + onSubmit(); + } + }} + /> + +
is / is not — phrase match. On a keyword field this is exact equality; on a text field it matches a contiguous sequence of tokens anywhere in the value.
+
contains / does not contain — single-token match (no whitespace allowed). On a keyword field this is exact equality; on a text field it matches any document whose analyzed tokens include this term. Not a substring match — “germ” will not match “germany”.
+
> / < — numeric or date range. Shown only for numeric and date fields.
+
exists / does not exist — presence check, no value needed.
+
+ } + placement="top" + > + + + + {!['exists', 'not exists'].includes(value.filter.operator) && ( + { + dispatch(changeFilterValue({ id: value.id, value: e.value ?? '' })); + onSubmit(); + }} + /> + )} + + + ); +}; diff --git a/src/components/QueryEditor/FilterEditor/state/actions.ts b/src/components/QueryEditor/FilterEditor/state/actions.ts new file mode 100644 index 0000000..ae9b7b5 --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/state/actions.ts @@ -0,0 +1,10 @@ +import { createAction } from '@reduxjs/toolkit'; + +import { QueryFilter } from '@/types'; + +export const addFilter = createAction('@filters/add'); +export const removeFilter = createAction('@filters/remove'); +export const toggleFilterVisibility = createAction('@filters/toggle_visibility'); +export const changeFilterField = createAction<{ id: QueryFilter['id']; field: string }>('@filters/change_field'); +export const changeFilterValue = createAction<{ id: QueryFilter['id']; value: string }>('@filters/change_value'); +export const changeFilterOperation = createAction<{ id: QueryFilter['id']; op: string }>('@filters/change_operation'); diff --git a/src/components/QueryEditor/FilterEditor/state/reducer.ts b/src/components/QueryEditor/FilterEditor/state/reducer.ts new file mode 100644 index 0000000..4ebc3b5 --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/state/reducer.ts @@ -0,0 +1,99 @@ +import { Action } from '@reduxjs/toolkit'; +import { defaultFilter } from '@/queryDef'; +import { ElasticsearchQuery } from '@/types'; +import { initExploreQuery, initQuery } from '../../state'; + +import { + addFilter, + changeFilterField, + changeFilterOperation, + changeFilterValue, + removeFilter, + toggleFilterVisibility, +} from './actions'; + +export const reducer = (state: ElasticsearchQuery['filters'], action: Action): ElasticsearchQuery['filters'] => { + + if (addFilter.match(action)) { + return [...state!, defaultFilter(action.payload)]; + } + + if (removeFilter.match(action)) { + const filterToRemove = state!.find((m) => m.id === action.payload)!; + const resultingFilters = state!.filter((filter) => filterToRemove.id !== filter.id); + if (resultingFilters.length === 0) { + return [defaultFilter()]; + } + return resultingFilters; + } + + if (changeFilterField.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + key: action.payload.field, + } + }; + }); + } + + if (changeFilterOperation.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + operator: action.payload.op, + } + }; + }); + } + + if (changeFilterValue.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + value: action.payload.value, + } + }; + }); + } + + if (toggleFilterVisibility.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload) { + return filter; + } + + return { + ...filter, + hide: !filter.hide, + }; + }); + } + + if (initQuery.match(action) || initExploreQuery.match(action)) { + if (state && state.length > 0) { + return state; + } + return [defaultFilter()]; + } + + return state; +}; diff --git a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx index e174716..2c8575a 100644 --- a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx +++ b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx @@ -24,6 +24,7 @@ describe('Metric Editor', () => { query: '', metrics: [avg], bucketAggs: [defaultBucketAgg('2')], + filters: [], }; const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); @@ -62,6 +63,7 @@ describe('Metric Editor', () => { query: '', metrics: [count], bucketAggs: [], + filters: [], }; const wrapper = ({ children }: PropsWithChildren<{}>) => ( diff --git a/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx index 5032c52..a0b26ad 100644 --- a/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx +++ b/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx @@ -27,6 +27,7 @@ describe('Settings Editor', () => { }, ], bucketAggs: [], + filters: [], }; const onChange = jest.fn(); @@ -102,6 +103,7 @@ describe('Settings Editor', () => { }, ], bucketAggs: [], + filters: [], }; const onChange = jest.fn(); diff --git a/src/components/QueryEditor/index.test.tsx b/src/components/QueryEditor/index.test.tsx index f51cf03..c932a8a 100644 --- a/src/components/QueryEditor/index.test.tsx +++ b/src/components/QueryEditor/index.test.tsx @@ -20,6 +20,7 @@ describe('QueryEditor', () => { ], // Even if present, this shouldn't be shown in the UI bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; render(); @@ -38,6 +39,7 @@ describe('QueryEditor', () => { }, ], bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; render(); diff --git a/src/components/QueryEditor/index.tsx b/src/components/QueryEditor/index.tsx index 773c435..9059fee 100644 --- a/src/components/QueryEditor/index.tsx +++ b/src/components/QueryEditor/index.tsx @@ -25,6 +25,7 @@ import { QueryTypeSelector } from './QueryTypeSelector'; import { getHook } from '@/utils/context'; import { LuceneQueryEditor } from '@/components/LuceneQueryEditor'; import { useDatasourceFields } from '@/datasource/utils'; +import { FilterEditor } from '@/components/QueryEditor/FilterEditor'; export type ElasticQueryEditorProps = QueryEditorProps; @@ -133,6 +134,7 @@ const QueryEditorForm = ({ value, onRunQuery }: Props) => { value={value?.query} onSubmit={onSubmitCB}/> + {showBucketAggregationsEditor && } diff --git a/src/configuration/ConfigEditor.tsx b/src/configuration/ConfigEditor.tsx index 511902d..5bac6d8 100644 --- a/src/configuration/ConfigEditor.tsx +++ b/src/configuration/ConfigEditor.tsx @@ -1,7 +1,7 @@ import React, { useCallback } from 'react'; -import { DataSourceHttpSettings, Input, InlineField, FieldSet } from '@grafana/ui'; -import { DataSourcePluginOptionsEditorProps, DataSourceSettings } from '@grafana/data'; -import { QuickwitOptions } from '../quickwit'; +import { DataSourceHttpSettings, Input, InlineField, FieldSet, RadioButtonGroup } from '@grafana/ui'; +import { DataSourcePluginOptionsEditorProps, DataSourceSettings, SelectableValue } from '@grafana/data'; +import { FilterAutocompleteChainMode, QuickwitOptions } from '../quickwit'; import { coerceOptions } from './utils'; import { Divider } from '../components/Divider'; import { DataLinks } from './DataLinks'; @@ -9,6 +9,12 @@ import _ from 'lodash'; interface Props extends DataSourcePluginOptionsEditorProps {} +const filterChainModeOptions: Array> = [ + { label: 'No chain', value: 'none' }, + { label: 'Sample', value: 'sample' }, + { label: 'Full', value: 'full' }, +]; + export const ConfigEditor = (props: Props) => { const { options: originalOptions, onOptionsChange } = props; const options = coerceOptions(originalOptions); @@ -103,6 +109,47 @@ export const QuickwitDetails = ({ value, onChange }: DetailsProps) => { width={40} /> + + + onChange({ + ...value, + jsonData: { ...value.jsonData, filterAutocompleteLimit: event.currentTarget.value }, + }) + } + placeholder="1000" + width={40} + /> + + + + onChange({ + ...value, + jsonData: { + ...value.jsonData, + filterAutocompleteChainMode: mode, + filterAutocompleteUseFilterChains: mode !== 'none', + }, + }) + } + /> + diff --git a/src/configuration/utils.ts b/src/configuration/utils.ts index 748c2d0..1a7c47d 100644 --- a/src/configuration/utils.ts +++ b/src/configuration/utils.ts @@ -4,12 +4,19 @@ import { QuickwitOptions } from 'quickwit'; export const coerceOptions = ( options: DataSourceSettings ): DataSourceSettings => { + const filterAutocompleteChainMode = + options.jsonData.filterAutocompleteChainMode ?? + (options.jsonData.filterAutocompleteUseFilterChains === false ? 'none' : 'sample'); + return { ...options, jsonData: { ...options.jsonData, logMessageField: options.jsonData.logMessageField || '', logLevelField: options.jsonData.logLevelField || '', + filterAutocompleteLimit: options.jsonData.filterAutocompleteLimit ?? '1000', + filterAutocompleteChainMode, + filterAutocompleteUseFilterChains: filterAutocompleteChainMode !== 'none', }, }; }; diff --git a/src/dataquery.gen.ts b/src/dataquery.gen.ts index bd5dc47..0b70b88 100644 --- a/src/dataquery.gen.ts +++ b/src/dataquery.gen.ts @@ -9,6 +9,7 @@ // Run 'make gen-cue' from repository root to regenerate. import { DataQuery } from '@grafana/schema'; +import { AdHocVariableFilter } from '@grafana/data'; export const DataQueryModelVersion = Object.freeze([0, 0]); @@ -128,6 +129,12 @@ export interface BaseMetricAggregation { type: MetricAggregationType; } +export interface QueryFilter { + hide?: boolean; + id: string; + filter: AdHocVariableFilter; +} + export interface PipelineVariable { name: string; pipelineAgg: string; @@ -392,6 +399,10 @@ export interface Elasticsearch extends DataQuery { * List of metric aggregations */ metrics?: MetricAggregation[]; + /** + * List of filters + */ + filters?: QueryFilter[]; /** * Lucene query */ @@ -405,4 +416,5 @@ export interface Elasticsearch extends DataQuery { export const defaultElasticsearch: Partial = { bucketAggs: [], metrics: [], + filters: [], }; diff --git a/src/datasource/base.test.ts b/src/datasource/base.test.ts index d6f268e..a4b9eff 100644 --- a/src/datasource/base.test.ts +++ b/src/datasource/base.test.ts @@ -1,4 +1,15 @@ -import { formatQuery, luceneEscape } from './base'; +import { AdHocVariableFilter } from '@grafana/data'; +import { from } from 'rxjs'; + +import { addAddHocFilter } from '../modifyQuery'; +import { ElasticsearchQuery } from '../types'; +import { + BaseQuickwitDataSource, + formatQuery, + luceneEscape, + parseFilterAutocompleteChainMode, + parseFilterAutocompleteLimit, +} from './base'; describe('BaseQuickwitDataSource', () => { describe('luceneEscape', () => { @@ -24,151 +35,674 @@ describe('BaseQuickwitDataSource', () => { }); describe('formatQuery', () => { + describe('String values', () => { + it('should return escaped string for simple string values', () => { + const result = formatQuery('simple_value', { id: 'test_var' }); + expect(result).toBe('simple_value'); + }); + + it('should escape special characters in string values', () => { + const result = formatQuery('value+with-special:chars', { id: 'test_var' }); + expect(result).toBe('value\\+with\\-special\\:chars'); + }); + + it('should not escape numeric strings', () => { + const result = formatQuery('123', { id: 'test_var' }); + expect(result).toBe('123'); + }); + }); + + describe('Array values with valid field configuration', () => { + const validVariable = { + id: 'test_var', + query: '{"field": "status"}' + }; + + it('should format array values with field-specific OR syntax', () => { + const result = formatQuery(['error', 'warning'], validVariable); + expect(result).toBe('"error" OR status:"warning"'); + }); + + it('should handle single-item arrays', () => { + const result = formatQuery(['error'], validVariable); + expect(result).toBe('"error"'); + }); + + it('should escape special characters in array values', () => { + const result = formatQuery(['error+critical', 'warning:high'], validVariable); + expect(result).toBe('"error\\+critical" OR status:"warning\\:high"'); + }); + }); + + describe('Array values without field configuration', () => { + it('should use OR syntax when variable.query is undefined', () => { + const variable = { id: 'test_var' }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); - describe('String values', () => { - it('should return escaped string for simple string values', () => { - const result = formatQuery('simple_value', { id: 'test_var' }); - expect(result).toBe('simple_value'); + it('should use OR syntax when variable.query is null', () => { + const variable = { id: 'test_var', query: null }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); + + it('should use OR syntax when variable.query contains invalid JSON', () => { + const variable = { id: 'test_var', query: 'not valid json' }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); + + it('should use OR syntax when variable.query is valid JSON but missing field', () => { + const variable = { id: 'test_var', query: '{"other": "value"}' }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); + + it('should use OR syntax when field is not a string', () => { + const variable = { id: 'test_var', query: '{"field": 123}' }; + const result = formatQuery(['error', 'warning'], variable); + expect(result).toBe('"error" OR "warning"'); + }); + + it('should infer field-specific OR syntax from the query string', () => { + const variable = { id: 'levels' }; + const result = formatQuery(['error', 'warning'], variable, 'severity_text:$levels'); + expect(result).toBe('"error" OR severity_text:"warning"'); + }); + + it('should infer field-specific OR syntax from braced variables in the query string', () => { + const variable = { id: 'services' }; + const result = formatQuery(['web', 'api'], variable, 'resource_attributes.service.name:${services}'); + expect(result).toBe('"web" OR resource_attributes.service.name:"api"'); + }); }); - it('should escape special characters in string values', () => { - const result = formatQuery('value+with-special:chars', { id: 'test_var' }); - expect(result).toBe('value\\+with\\-special\\:chars'); + describe('Empty arrays', () => { + it('should return __empty__ for empty arrays', () => { + const variable = { id: 'test_var', query: '{"field": "status"}' }; + const result = formatQuery([], variable); + expect(result).toBe('__empty__'); + }); + + it('should return __empty__ for empty arrays even without field config', () => { + const variable = { id: 'test_var' }; + const result = formatQuery([], variable); + expect(result).toBe('__empty__'); + }); + }); + + describe('Error handling and robustness', () => { + it('should not throw when variable.query is undefined', () => { + expect(() => { + formatQuery(['test'], { id: 'test_var' }); + }).not.toThrow(); + }); + + it('should not throw when variable.query is malformed JSON', () => { + expect(() => { + formatQuery(['test'], { id: 'test_var', query: '{invalid json}' }); + }).not.toThrow(); + }); + + it('should not throw when variable.query is empty string', () => { + expect(() => { + formatQuery(['test'], { id: 'test_var', query: '' }); + }).not.toThrow(); + }); + + it('should handle non-string, non-array values', () => { + const result = formatQuery(123 as any, { id: 'test_var' }); + expect(result).toBe('123'); + }); + + it('should handle boolean values', () => { + const result = formatQuery(true as any, { id: 'test_var' }); + expect(result).toBe('true'); + }); }); - it('should not escape numeric strings', () => { - const result = formatQuery('123', { id: 'test_var' }); - expect(result).toBe('123'); + describe('Real-world scenarios', () => { + it('should handle variables from template variable queries', () => { + // Simulates a properly configured template variable + const templateVariable = { + id: 'log_level', + query: '{"field": "level"}' + }; + const result = formatQuery(['ERROR', 'WARN', 'INFO'], templateVariable); + expect(result).toBe('"ERROR" OR level:"WARN" OR level:"INFO"'); + }); + + it('should handle variables without query configuration (legacy/simple variables)', () => { + // Simulates a simple template variable without field configuration + const simpleVariable = { + id: 'service_names' + }; + const result = formatQuery(['web-service', 'api-service'], simpleVariable); + expect(result).toBe('"web\\-service" OR "api\\-service"'); + }); + + it('should handle variables with corrupted configuration', () => { + // Simulates a variable with corrupted/invalid configuration + const corruptedVariable = { + id: 'corrupted_var', + query: '{"field": undefined}' // Invalid JSON that might come from UI bugs + }; + const result = formatQuery(['value1', 'value2'], corruptedVariable); + expect(result).toBe('"value1" OR "value2"'); + }); }); }); - describe('Array values with valid field configuration', () => { - const validVariable = { - id: 'test_var', - query: '{"field": "status"}' + describe('quick filters', () => { + const addFilterToQuery = ( + fieldTypes: Record, + query: ElasticsearchQuery, + key: string, + value: string, + negate = false + ) => { + return (BaseQuickwitDataSource.prototype as any).addFilterToQuery.call( + Object.assign(Object.create(BaseQuickwitDataSource.prototype), { fieldTypes }), + query, + key, + value, + negate + ) as ElasticsearchQuery; }; - it('should format array values with field-specific OR syntax', () => { - const result = formatQuery(['error', 'warning'], validVariable); - expect(result).toBe('"error" OR status:"warning"'); + const renderAdHocFilters = ( + fieldTypes: Record, + filters: AdHocVariableFilter[] + ) => { + return (BaseQuickwitDataSource.prototype as any).addAdHocFilters.call( + { fieldTypes }, + '', + filters + ) as string; + }; + + const datasourceContext = (overrides: Record) => + Object.assign(Object.create(BaseQuickwitDataSource.prototype), overrides); + + it('adds text filters with whitespace as phrase filters', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; + + const updatedQuery = addFilterToQuery( + { 'attributes.grpc_message': 'text' }, + query, + 'attributes.grpc_message', + 'Error:[(0) invalid token, ]' + ); + + expect(updatedQuery.filters?.[0].filter).toEqual({ + key: 'attributes.grpc_message', + operator: '=', + value: 'Error:[(0) invalid token, ]', + }); }); - it('should handle single-item arrays', () => { - const result = formatQuery(['error'], validVariable); - expect(result).toBe('"error"'); + it('renders text phrase filters with quoted Quickwit syntax', () => { + const result = renderAdHocFilters( + { 'attributes.grpc_message': 'text' }, + [{ + key: 'attributes.grpc_message', + operator: '=', + value: 'Error:[(0) invalid token, ]', + }] + ); + + expect(result).toBe('attributes.grpc_message:"Error:[(0) invalid token, ]"'); }); - it('should escape special characters in array values', () => { - const result = formatQuery(['error+critical', 'warning:high'], validVariable); - expect(result).toBe('"error\\+critical" OR status:"warning\\:high"'); + it('renders negative text phrase filters with quoted Quickwit syntax', () => { + const result = renderAdHocFilters( + { 'attributes.grpc_message': 'text' }, + [{ + key: 'attributes.grpc_message', + operator: '!=', + value: 'Error:[(0) invalid token, ]', + }] + ); + + expect(result).toBe('-attributes.grpc_message:"Error:[(0) invalid token, ]"'); }); - }); - describe('Array values without field configuration', () => { - it('should use IN syntax when variable.query is undefined', () => { - const variable = { id: 'test_var' }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); + it('keeps simple-token text filters as term filters', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; + + const updatedQuery = addFilterToQuery( + { 'attributes.grpc_message': 'text' }, + query, + 'attributes.grpc_message', + 'unavailable' + ); + + expect(updatedQuery.filters?.[0].filter.operator).toBe('term'); }); - it('should use IN syntax when variable.query is null', () => { - const variable = { id: 'test_var', query: null }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); + it('reuses trailing empty filters without mutating the original query', () => { + const query = { + refId: 'A', + query: '', + metrics: [], + bucketAggs: [], + filters: [{ id: 'empty', filter: { key: '', operator: '=', value: '' } }], + } as any; + + const updatedQuery = addFilterToQuery( + { service_name: 'text' }, + query, + 'service_name', + 'frontend' + ); + + expect(updatedQuery).not.toBe(query); + expect(updatedQuery.filters?.[0]).toEqual({ + id: 'empty', + hide: false, + filter: { key: 'service_name', operator: 'term', value: 'frontend' }, + }); + expect(query.filters[0].filter).toEqual({ key: '', operator: '=', value: '' }); }); - it('should use IN syntax when variable.query contains invalid JSON', () => { - const variable = { id: 'test_var', query: 'not valid json' }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); + it('adds quick filters when filters are undefined', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [] } as any; + + const updatedQuery = addFilterToQuery( + { service_name: 'text' }, + query, + 'service_name', + 'frontend' + ); + + expect(updatedQuery.filters).toEqual([ + expect.objectContaining({ + hide: false, + filter: { key: 'service_name', operator: 'term', value: 'frontend' }, + }), + ]); }); - it('should use IN syntax when variable.query is valid JSON but missing field', () => { - const variable = { id: 'test_var', query: '{"other": "value"}' }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); + it('keeps JSON array text filters as equality filters', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; + + const updatedQuery = addFilterToQuery( + { 'attributes.tags': 'text' }, + query, + 'attributes.tags', + '["paperclip"]' + ); + + expect(updatedQuery.filters?.[0].filter).toEqual({ + key: 'attributes.tags', + operator: '=', + value: '["paperclip"]', + }); }); - it('should use IN syntax when field is not a string', () => { - const variable = { id: 'test_var', query: '{"field": 123}' }; - const result = formatQuery(['error', 'warning'], variable); - expect(result).toBe('IN ["error" "warning"]'); + it('renders JSON array text filters with Quickwit array syntax', () => { + const result = renderAdHocFilters( + { 'attributes.tags': 'text' }, + [{ + key: 'attributes.tags', + operator: '=', + value: '["paperclip","stapler"]', + }] + ); + + expect(result).toBe('attributes.tags:IN ["paperclip" "stapler"]'); }); - }); - describe('Empty arrays', () => { - it('should return __empty__ for empty arrays', () => { - const variable = { id: 'test_var', query: '{"field": "status"}' }; - const result = formatQuery([], variable); - expect(result).toBe('__empty__'); + it('keeps punctuated text filters as phrase filters', () => { + const query = { refId: 'A', query: '', metrics: [], bucketAggs: [], filters: [] } as any; + + const updatedQuery = addFilterToQuery( + { service_name: 'text' }, + query, + 'service_name', + 'auth-api' + ); + + expect(updatedQuery.filters?.[0].filter).toEqual({ + key: 'service_name', + operator: '=', + value: 'auth-api', + }); }); - it('should return __empty__ for empty arrays even without field config', () => { - const variable = { id: 'test_var' }; - const result = formatQuery([], variable); - expect(result).toBe('__empty__'); + it('renders punctuated text filters with quoted Quickwit syntax', () => { + const result = renderAdHocFilters( + { service_name: 'text' }, + [{ + key: 'service_name', + operator: '=', + value: 'auth-api', + }] + ); + + expect(result).toBe('service_name:"auth-api"'); }); - }); - describe('Error handling and robustness', () => { - it('should not throw when variable.query is undefined', () => { - expect(() => { - formatQuery(['test'], { id: 'test_var' }); - }).not.toThrow(); + it('escapes special characters in unquoted term filters', () => { + const result = addAddHocFilter('', { + key: 'attributes.grpc_message', + operator: 'term', + value: 'error:foo', + }); + + expect(result).toBe('attributes.grpc_message:error\\:foo'); }); - it('should not throw when variable.query is malformed JSON', () => { - expect(() => { - formatQuery(['test'], { id: 'test_var', query: '{invalid json}' }); - }).not.toThrow(); + it('toggles off matching quick filters by operator', () => { + const query = { + refId: 'A', + query: '', + metrics: [], + bucketAggs: [], + filters: [{ id: 'existing', filter: { key: 'service_name', operator: 'term', value: 'frontend' } }], + } as any; + + const updatedQuery = (BaseQuickwitDataSource.prototype as any).toggleQueryFilter.call( + datasourceContext({ fieldTypes: { service_name: 'text' } }), + query, + { type: 'FILTER_FOR', options: { key: 'service_name', value: 'frontend' } } + ); + + expect(updatedQuery.filters).toEqual([{ id: expect.any(String), filter: { key: '', operator: '=', value: '' } }]); }); - it('should not throw when variable.query is empty string', () => { - expect(() => { - formatQuery(['test'], { id: 'test_var', query: '' }); - }).not.toThrow(); + it('replaces opposite quick filters instead of adding duplicates', () => { + const query = { + refId: 'A', + query: '', + metrics: [], + bucketAggs: [], + filters: [{ id: 'existing', filter: { key: 'service_name', operator: 'not term', value: 'frontend' } }], + } as any; + + const updatedQuery = (BaseQuickwitDataSource.prototype as any).toggleQueryFilter.call( + datasourceContext({ fieldTypes: { service_name: 'text' } }), + query, + { type: 'FILTER_FOR', options: { key: 'service_name', value: 'frontend' } } + ); + + expect(updatedQuery.filters).toEqual([ + { id: 'existing', hide: false, filter: { key: 'service_name', operator: 'term', value: 'frontend' } }, + ]); }); - it('should handle non-string, non-array values', () => { - const result = formatQuery(123 as any, { id: 'test_var' }); - expect(result).toBe('123'); + it('can check quick filters by operator when Grafana provides filter direction', () => { + const query = { + refId: 'A', + query: '', + metrics: [], + bucketAggs: [], + filters: [{ id: 'existing', filter: { key: 'service_name', operator: 'not term', value: 'frontend' } }], + } as any; + const datasource = datasourceContext({ fieldTypes: { service_name: 'text' } }); + + expect((BaseQuickwitDataSource.prototype as any).queryHasFilter.call( + datasource, + query, + { key: 'service_name', value: 'frontend', type: 'FILTER_FOR' } + )).toBe(false); + expect((BaseQuickwitDataSource.prototype as any).queryHasFilter.call( + datasource, + query, + { key: 'service_name', value: 'frontend', type: 'FILTER_OUT' } + )).toBe(true); }); - it('should handle boolean values', () => { - const result = formatQuery(true as any, { id: 'test_var' }); - expect(result).toBe('true'); + it('applies prior filters when loading tag values', async () => { + const getTerms = jest.fn(() => from([[]])); + + await (BaseQuickwitDataSource.prototype as any).getTagValues.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteLimit: 1000, + filterAutocompleteUseFilterChains: true, + getTerms, + }), + { + key: 'status', + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(getTerms).toHaveBeenCalledWith( + { field: 'status', query: 'service:"frontend"', size: 1000 }, + undefined + ); }); - }); - describe('Real-world scenarios', () => { - it('should handle variables from template variable queries', () => { - // Simulates a properly configured template variable - const templateVariable = { - id: 'log_level', - query: '{"field": "level"}' - }; - const result = formatQuery(['ERROR', 'WARN', 'INFO'], templateVariable); - expect(result).toBe('"ERROR" OR level:"WARN" OR level:"INFO"'); + it('uses the datasource autocomplete limit when loading tag values', async () => { + const getTerms = jest.fn(() => from([[]])); + + await (BaseQuickwitDataSource.prototype as any).getTagValues.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteLimit: 250, + filterAutocompleteUseFilterChains: true, + getTerms, + }), + { + key: 'status', + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(getTerms).toHaveBeenCalledWith( + { field: 'status', query: 'service:"frontend"', size: 250 }, + undefined + ); }); - it('should handle variables without query configuration (legacy/simple variables)', () => { - // Simulates a simple template variable without field configuration - const simpleVariable = { - id: 'service_names' - }; - const result = formatQuery(['web-service', 'api-service'], simpleVariable); - expect(result).toBe('IN ["web\\-service" "api\\-service"]'); + it('can disable filter chains for tag values', async () => { + const getTerms = jest.fn(() => from([[]])); + + await (BaseQuickwitDataSource.prototype as any).getTagValues.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteLimit: 1000, + filterAutocompleteUseFilterChains: false, + getTerms, + }), + { + key: 'status', + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(getTerms).toHaveBeenCalledWith( + { field: 'status', query: '', size: 1000 }, + undefined + ); }); - it('should handle variables with corrupted configuration', () => { - // Simulates a variable with corrupted/invalid configuration - const corruptedVariable = { - id: 'corrupted_var', - query: '{"field": undefined}' // Invalid JSON that might come from UI bugs - }; - const result = formatQuery(['value1', 'value2'], corruptedVariable); - expect(result).toBe('IN ["value1" "value2"]'); + it('derives chained tag keys from fields present in matching documents', async () => { + const getFields = jest.fn(() => + from([[ + { text: 'service', type: 'string' }, + { text: 'status', type: 'string' }, + { text: 'missing', type: 'string' }, + ]]) + ); + const query = jest.fn(() => + from([{ + data: [ + { + fields: [ + { name: 'service' }, + { name: 'status' }, + { name: 'sort' }, + ], + }, + ], + }]) + ); + + const result = await (BaseQuickwitDataSource.prototype as any).getTagKeys.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteLimit: 5, + filterAutocompleteUseFilterChains: true, + getFields, + query, + }), + { + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(result).toEqual([ + { text: 'service', type: 'string' }, + { text: 'status', type: 'string' }, + ]); + expect(query).toHaveBeenCalledWith( + expect.objectContaining({ + requestId: expect.stringMatching(/^getFilterKeys-/), + targets: [ + expect.objectContaining({ + query: 'service:"frontend"', + metrics: [{ id: 'filterKeys', type: 'raw_data', settings: { size: '5' } }], + }), + ], + }) + ); + }); + + it('derives full chained tag keys from every matching document', async () => { + const getFields = jest.fn(() => + from([[ + { text: 'service', type: 'string' }, + { text: 'status', type: 'string' }, + { text: 'rare.field', type: 'string' }, + { text: 'missing', type: 'string' }, + ]]) + ); + const postResource = jest.fn() + .mockResolvedValueOnce({ + num_hits: 2, + hits: [{ service: 'frontend', status: 'ok' }], + }) + .mockResolvedValueOnce({ + num_hits: 2, + hits: [{ service: 'frontend', rare: { field: 'present' } }], + }); + + const result = await (BaseQuickwitDataSource.prototype as any).getTagKeys.call( + datasourceContext({ + fieldTypes: {}, + filterAutocompleteChainMode: 'full', + getFields, + index: 'logs', + postResource, + }), + { + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(result).toEqual([ + { text: 'service', type: 'string' }, + { text: 'status', type: 'string' }, + { text: 'rare.field', type: 'string' }, + ]); + expect(postResource).toHaveBeenCalledTimes(2); + expect(postResource).toHaveBeenNthCalledWith( + 1, + 'indexes/logs/search', + expect.objectContaining({ + query: 'service:"frontend"', + max_hits: 1000, + start_offset: 0, + }), + expect.objectContaining({ requestId: expect.stringMatching(/^getFilterKeysFull-/) }) + ); + expect(postResource).toHaveBeenNthCalledWith( + 2, + 'indexes/logs/search', + expect.objectContaining({ + query: 'service:"frontend"', + max_hits: 1000, + start_offset: 1, + }), + expect.objectContaining({ requestId: expect.stringMatching(/^getFilterKeysFull-/) }) + ); + }); + + it('deduplicates tag keys that have multiple field capabilities', async () => { + const getFields = jest.fn(() => + from([[ + { text: 'service_name', type: 'string' }, + { text: 'service_name', type: 'keyword' }, + { text: 'severity_text', type: 'string' }, + ]]) + ); + + const result = await (BaseQuickwitDataSource.prototype as any).getTagKeys.call( + datasourceContext({ + filterAutocompleteUseFilterChains: true, + getFields, + }), + {} + ); + + expect(result).toEqual([ + { text: 'service_name', type: 'string' }, + { text: 'severity_text', type: 'string' }, + ]); + }); + + it('can disable filter chains for tag keys', async () => { + const getFields = jest.fn(() => from([[{ text: 'service', type: 'string' }]])); + const query = jest.fn(); + + const result = await (BaseQuickwitDataSource.prototype as any).getTagKeys.call( + datasourceContext({ + filterAutocompleteUseFilterChains: false, + getFields, + query, + }), + { + filters: [{ key: 'service', operator: '=', value: 'frontend' }], + } + ); + + expect(result).toEqual([{ text: 'service', type: 'string' }]); + expect(query).not.toHaveBeenCalled(); }); }); + + describe('filter autocomplete limit', () => { + it('defaults invalid and empty values to 1000', () => { + expect(parseFilterAutocompleteLimit(undefined)).toBe(1000); + expect(parseFilterAutocompleteLimit('')).toBe(1000); + expect(parseFilterAutocompleteLimit('invalid')).toBe(1000); + expect(parseFilterAutocompleteLimit('-1')).toBe(1000); + }); + + it('accepts positive values and zero', () => { + expect(parseFilterAutocompleteLimit('250')).toBe(250); + expect(parseFilterAutocompleteLimit('0')).toBe(0); + }); + }); + + describe('filter autocomplete chain mode', () => { + it('defaults missing and invalid modes to sample', () => { + expect(parseFilterAutocompleteChainMode(undefined)).toBe('sample'); + expect(parseFilterAutocompleteChainMode('invalid')).toBe('sample'); + }); + + it('accepts supported modes', () => { + expect(parseFilterAutocompleteChainMode('none')).toBe('none'); + expect(parseFilterAutocompleteChainMode('sample')).toBe('sample'); + expect(parseFilterAutocompleteChainMode('full')).toBe('full'); + }); + + it('maps legacy disabled filter chains to none', () => { + expect(parseFilterAutocompleteChainMode(undefined, false)).toBe('none'); + }); }); }); diff --git a/src/datasource/base.ts b/src/datasource/base.ts index e90f47d..bce19df 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -14,13 +14,14 @@ import { QueryFixAction, ScopedVars, TimeRange, + ToggleFilterAction, } from '@grafana/data'; import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery, FieldCapabilitiesResponse } from '@/types'; -import { - DataSourceWithBackend, - getTemplateSrv, +import { + DataSourceWithBackend, + getTemplateSrv, TemplateSrv } from '@grafana/runtime'; -import { QuickwitOptions } from 'quickwit'; +import { FilterAutocompleteChainMode, QuickwitOptions } from 'quickwit'; import { getDataQuery } from 'QueryBuilder/elastic'; import { metricAggregationConfig } from 'components/QueryEditor/MetricAggregationsEditor/utils'; @@ -28,19 +29,46 @@ import { isMetricAggregationWithField } from 'components/QueryEditor/MetricAggre import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregationsEditor/utils'; import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations'; import ElasticsearchLanguageProvider from 'LanguageProvider'; -import { fieldTypeMap } from 'utils'; +import { fieldTypeMap, hasWhiteSpace, isSimpleToken } from 'utils'; import { addAddHocFilter } from 'modifyQuery'; import { getQueryResponseProcessor } from 'datasource/processResponse'; import { SECOND } from 'utils/time'; import { GConstructor } from 'utils/mixins'; -import { LuceneQuery } from '@/utils/lucene'; -import { uidMaker } from "@/utils/uid" +import { newFilterId, uidMaker } from '@/utils/uid'; import { DefaultsConfigOverrides } from 'store/defaults/conf'; +import { isSet } from '@/utils'; export type BaseQuickwitDataSourceConstructor = GConstructor -const getQueryUid = uidMaker("query") +const getQueryUid = uidMaker('query'); +export const DEFAULT_FILTER_AUTOCOMPLETE_LIMIT = 1000; +const DEFAULT_FILTER_AUTOCOMPLETE_CHAIN_MODE: FilterAutocompleteChainMode = 'sample'; +const FULL_FILTER_CHAIN_PAGE_SIZE = 1000; + +export function parseFilterAutocompleteLimit(value: unknown): number { + if (value === undefined || value === null || value === '') { + return DEFAULT_FILTER_AUTOCOMPLETE_LIMIT; + } + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 0) { + return DEFAULT_FILTER_AUTOCOMPLETE_LIMIT; + } + return Math.floor(parsed); +} + +export function parseFilterAutocompleteChainMode( + value: unknown, + legacyUseFilterChains?: boolean +): FilterAutocompleteChainMode { + if (value === 'none' || value === 'sample' || value === 'full') { + return value; + } + if (legacyUseFilterChains === false) { + return 'none'; + } + return DEFAULT_FILTER_AUTOCOMPLETE_CHAIN_MODE; +} type FieldCapsSpec = { aggregatable?: boolean, @@ -49,6 +77,11 @@ type FieldCapsSpec = { range?: TimeRange } +type QuickwitSearchResponse = { + num_hits?: number, + hits?: Array>, +} + export class BaseQuickwitDataSource extends DataSourceWithBackend implements @@ -62,7 +95,16 @@ export class BaseQuickwitDataSource queryEditorConfig?: { defaults?: DefaultsConfigOverrides }; + filterAutocompleteLimit: number; + filterAutocompleteChainMode: FilterAutocompleteChainMode; languageProvider: ElasticsearchLanguageProvider; + // Populated lazily by getFields(). Used by modifyQuery to pick an operator + // that matches text field semantics. + private fieldTypes: Record = {}; + + getFieldType(name: string): string | undefined { + return this.fieldTypes[name]; + } constructor( @@ -77,8 +119,19 @@ export class BaseQuickwitDataSource this.logLevelField = settingsData.logLevelField || ''; this.dataLinks = settingsData.dataLinks || []; this.queryEditorConfig = settingsData.queryEditorConfig || {}; + this.filterAutocompleteLimit = parseFilterAutocompleteLimit(settingsData.filterAutocompleteLimit); + this.filterAutocompleteChainMode = parseFilterAutocompleteChainMode( + settingsData.filterAutocompleteChainMode, + settingsData.filterAutocompleteUseFilterChains + ); this.languageProvider = new ElasticsearchLanguageProvider(this); this.annotations = {}; + + // Warm the schema cache so modifyQuery can pick the right operator even when + // the query editor hasn't mounted (e.g. dashboard view mode). Fire-and-forget. + if (this.index) { + lastValueFrom(this.getFields()).catch(() => {}); + } } query(request: DataQueryRequest): Observable { @@ -121,32 +174,130 @@ export class BaseQuickwitDataSource if (!action.options) { return query; } + if (action.type !== 'ADD_FILTER' && action.type !== 'ADD_FILTER_OUT') { + console.warn('unsupported operation', action.type); + return query; + } - let lquery = LuceneQuery.parse(query.query ?? '') - switch (action.type) { - case 'ADD_FILTER': { - lquery = lquery.addFilter(action.options.key, action.options.value) - break; - } - case 'ADD_FILTER_OUT': { - lquery = lquery.addFilter(action.options.key, action.options.value, '-') - break; + const key = action.options.key; + const rawValue = String(action.options.value ?? ''); + const negate = action.type === 'ADD_FILTER_OUT'; + return this.addFilterToQuery(query, key, rawValue, negate); + } + + /** + * Newer Grafana interface (DataSourceWithToggleableQueryFiltersSupport) used + * by the Logs panel in dashboards. Preferred over modifyQuery when present. + */ + toggleQueryFilter(query: ElasticsearchQuery, filter: ToggleFilterAction): ElasticsearchQuery { + const key = filter.options.key; + const rawValue = String(filter.options.value ?? ''); + const negate = filter.type === 'FILTER_OUT'; + const operator = this.getFilterOperator(key, rawValue, negate); + const oppositeOperator = this.getFilterOperator(key, rawValue, !negate); + const emptyFilter = () => [{ id: newFilterId(), filter: { key: '', operator: '=', value: '' } }]; + const filters = query.filters ?? []; + + // If the same filter is already present, toggle it off. + const existingIdx = filters.findIndex( + (f) => !f.hide && f.filter.key === key && f.filter.value === rawValue && f.filter.operator === operator + ); + if (existingIdx !== -1) { + const next = [...filters]; + next.splice(existingIdx, 1); + return { ...query, filters: next.length ? next : emptyFilter() }; + } + + // If the opposite filter is present, replace it. + const oppositeIdx = filters.findIndex( + (f) => !f.hide && f.filter.key === key && f.filter.value === rawValue && f.filter.operator === oppositeOperator + ); + if (oppositeIdx !== -1) { + return { + ...query, + filters: filters.map((existing, index) => + index === oppositeIdx + ? { ...existing, hide: false, filter: { key, operator, value: rawValue } } + : existing + ), + }; + } + + return this.addFilterToQuery(query, key, rawValue, negate); + } + + queryHasFilter(query: ElasticsearchQuery, filter: { key: string; value: string; operator?: string; type?: string }): boolean { + const expectedOperator = filter.operator || ( + filter.type === 'FILTER_FOR' || filter.type === 'FILTER_OUT' + ? this.getFilterOperator(filter.key, String(filter.value ?? ''), filter.type === 'FILTER_OUT') + : undefined + ); + return (query.filters ?? []).some( + (f) => + f.filter.key === filter.key && + f.filter.value === filter.value && + !f.hide && + (expectedOperator === undefined || f.filter.operator === expectedOperator) + ); + } + + private getFilterOperator(key: string, value: string, negate: boolean): string { + const fieldType = this.fieldTypes[key]; + const isText = fieldType === 'text'; + + const useTermOperator = isText && value !== '' && isSimpleToken(value); + return useTermOperator + ? (negate ? 'not term' : 'term') + : (negate ? '!=' : '='); + } + + private addFilterToQuery(query: ElasticsearchQuery, key: string, rawValue: string, negate: boolean): ElasticsearchQuery { + const operator = this.getFilterOperator(key, rawValue, negate); + const filters = query.filters ?? []; + const nextFilter = { key, operator, value: rawValue }; + + // If the user hasn't populated any filter yet, reuse the trailing empty one. + const len = filters.length; + if (len > 0) { + const last = filters[len - 1]; + if (!isSet(last.filter.key) && !isSet(last.filter.value)) { + return { + ...query, + filters: filters.map((filter, index) => + index === len - 1 ? { ...filter, hide: false, filter: nextFilter } : filter + ), + }; } } - return { ...query, query: lquery.toString() }; + + return { + ...query, + filters: [ + ...filters, + { + id: newFilterId(), + hide: false, + filter: nextFilter, + }, + ], + }; } getDataQueryRequest(queryDef: TermsQuery, range: TimeRange, requestId?: string) { let dataQuery = getDataQuery(queryDef, 'getTerms'); + return this.getRequestForQuery(dataQuery, range, requestId); + } + + private getRequestForQuery(query: ElasticsearchQuery, range: TimeRange, requestId?: string) { const request: DataQueryRequest = { app: CoreApp.Unknown, requestId: requestId || getQueryUid.next(), interval: '', intervalMs: 0, range, - targets:[dataQuery], - timezone:'browser', - scopedVars:{}, + targets: [query], + timezone: 'browser', + scopedVars: {}, startTime: Date.now(), } return request @@ -174,6 +325,18 @@ export class BaseQuickwitDataSource end_timestamp: Math.ceil(range.to.valueOf()/SECOND), })).pipe( map((field_capabilities_response: FieldCapabilitiesResponse) => { + // Cache field → type on the datasource for modifyQuery to consult. + // Quickwit routes phrase queries to the text variant first on multi-indexed + // fields (text+keyword), and text fields don't index positions by default. + // So prefer 'text' when present — it drives safer operator choices downstream. + for (const [name, caps] of Object.entries(field_capabilities_response.fields)) { + const typeKeys = Object.keys(caps); + const chosen = typeKeys.includes('text') ? 'text' : typeKeys[0]; + if (chosen) { + this.fieldTypes[name] = chosen; + } + } + const shouldAddField = (field: any) => { if (spec.aggregatable !== undefined && field.aggregatable !== spec.aggregatable) { return false @@ -198,7 +361,7 @@ export class BaseQuickwitDataSource .map(field_capability => { return { text: field_capability.field_name, - type: fieldTypeMap[field_capability.type], + type: fieldTypeMap[field_capability.type], } }); const uniquefieldCapabilities = fieldCapabilities.filter((field_capability, index, self) => @@ -214,17 +377,136 @@ export class BaseQuickwitDataSource /** * Get tag keys for adhoc filters */ - getTagKeys(options: any) { - const fields = this.getFields({aggregatable:true, range: options.timeRange}) - return lastValueFrom(fields, {defaultValue:[]}); + getTagKeys(options: any = {}) { + const fieldSpec = this.getTagKeyFieldSpec(options); + const filters = this.getFilterChain(options.filters); + if (!filters?.length) { + return lastValueFrom(this.getFields(fieldSpec), { defaultValue: [] }).then((fields) => this.dedupeTagKeys(fields)); + } + if (this.getFilterAutocompleteChainMode() === 'full') { + return this.getFullFilteredTagKeys(filters, fieldSpec); + } + return this.getSampledFilteredTagKeys(filters, fieldSpec); } /** * Get tag values for adhoc filters */ getTagValues(options: any) { - const terms = this.getTerms({ field: options.key }, options.timeRange) - return lastValueFrom(terms, {defaultValue:[]}); + const query = this.addAdHocFilters('', this.getFilterChain(options.filters)); + const terms = this.getTerms({ field: options.key, query, size: this.filterAutocompleteLimit }, options.timeRange); + return lastValueFrom(terms, { defaultValue: [] }); + } + + private getFilterChain(filters?: AdHocVariableFilter[]) { + return this.getFilterAutocompleteChainMode() === 'none' ? undefined : filters; + } + + private getFilterAutocompleteChainMode() { + return parseFilterAutocompleteChainMode( + this.filterAutocompleteChainMode, + (this as { filterAutocompleteUseFilterChains?: boolean }).filterAutocompleteUseFilterChains + ); + } + + private getTagKeyFieldSpec(options: any = {}): FieldCapsSpec { + const spec: FieldCapsSpec = { range: options.timeRange }; + if (options.aggregatable !== undefined) { + spec.aggregatable = options.aggregatable; + } else if (options.searchable === undefined) { + spec.aggregatable = true; + } + if (options.searchable !== undefined) { + spec.searchable = options.searchable; + } + if (options.type !== undefined) { + spec.type = options.type; + } + return spec; + } + + private dedupeTagKeys(fields: MetricFindValue[]) { + const seen = new Set(); + return fields.filter((field) => { + const name = String(field.text); + if (seen.has(name)) { + return false; + } + seen.add(name); + return true; + }); + } + + private async getSampledFilteredTagKeys(filters: AdHocVariableFilter[], fieldSpec: FieldCapsSpec) { + const query = this.addAdHocFilters('', filters); + // Field-capabilities are schema-wide. To approximate chained field-key + // suggestions, sample matching documents and keep fields present in that + // sample. A limit of 0 means "no terms limit" elsewhere, but raw-document + // sampling still needs a finite size. + const sampleLimit = this.filterAutocompleteLimit > 0 ? this.filterAutocompleteLimit : DEFAULT_FILTER_AUTOCOMPLETE_LIMIT; + const target: ElasticsearchQuery = { + refId: 'filterKeys', + query, + metrics: [{ id: 'filterKeys', type: 'raw_data', settings: { size: sampleLimit.toString() } }], + bucketAggs: [], + filters: [], + }; + const range = fieldSpec.range || getDefaultTimeRange(); + const [fields, response] = await Promise.all([ + lastValueFrom(this.getFields(fieldSpec), { defaultValue: [] }), + lastValueFrom(this.query(this.getRequestForQuery(target, range, `getFilterKeys-${getQueryUid.next()}`)), { + defaultValue: { data: [] }, + }), + ]); + const presentFields = new Set(); + response.data?.forEach((frame: DataFrame) => { + frame.fields?.forEach((field: { name?: string }) => { + if (field.name && field.name !== 'sort') { + presentFields.add(field.name); + } + }); + }); + return this.dedupeTagKeys(fields).filter((field) => presentFields.has(String(field.text))); + } + + private async getFullFilteredTagKeys(filters: AdHocVariableFilter[], fieldSpec: FieldCapsSpec) { + const query = this.addAdHocFilters('', filters); + const range = fieldSpec.range || getDefaultTimeRange(); + const [fields, presentFields] = await Promise.all([ + lastValueFrom(this.getFields(fieldSpec), { defaultValue: [] }), + this.getAllMatchingFieldNames(query, range), + ]); + return this.dedupeTagKeys(fields).filter((field) => presentFields.has(String(field.text))); + } + + private async getAllMatchingFieldNames(query: string, range: TimeRange) { + const presentFields = new Set(); + let offset = 0; + let totalHits: number | undefined; + + do { + const response = await this.postResource( + `indexes/${this.index}/search`, + { + query: query || '*', + max_hits: FULL_FILTER_CHAIN_PAGE_SIZE, + start_offset: offset, + start_timestamp: Math.floor(range.from.valueOf() / SECOND), + end_timestamp: Math.ceil(range.to.valueOf() / SECOND), + }, + { requestId: `getFilterKeysFull-${getQueryUid.next()}` } + ); + const hits = response.hits ?? []; + totalHits = response.num_hits ?? totalHits ?? hits.length; + + hits.forEach((hit) => collectFieldNames(hit, presentFields)); + offset += hits.length; + if (hits.length === 0) { + break; + } + } while (totalHits === undefined || offset < totalHits); + + return presentFields; } /** @@ -308,7 +590,9 @@ export class BaseQuickwitDataSource } interpolateLuceneQuery(queryString: string, scopedVars?: ScopedVars) { - return this.templateSrv.replace(queryString, scopedVars, formatQuery); + return this.templateSrv.replace(queryString, scopedVars, (value: string | string[], variable: any) => + formatQuery(value, variable, queryString) + ); } interpolateVariablesInQueries(queries: ElasticsearchQuery[], scopedVars: ScopedVars | {}, filters?: AdHocVariableFilter[]): ElasticsearchQuery[] { @@ -336,10 +620,31 @@ export class BaseQuickwitDataSource return bucketAgg; }; + const renderedQuery = (() => { + let q = this.interpolateLuceneQuery(query.query || '', scopedVars); + const queryFilters = query.filters + ?.filter((f) => { + if (f.hide) { + return false; + } + const hasValidValue = ( + ['exists', 'not exists'].includes(f.filter.operator) || isSet(f.filter.value) + ) && ( + !['term', 'not term'].includes(f.filter.operator) || !hasWhiteSpace(f.filter.value) + ) + + return isSet(f.filter.key) && hasValidValue && isSet(f.filter.operator) + }) + .map((f) => f.filter); + q = this.addAdHocFilters(q, queryFilters) + q = this.addAdHocFilters(q, filters) + return q + })() + const expandedQuery = { ...query, datasource: this.getRef(), - query: this.addAdHocFilters(this.interpolateLuceneQuery(query.query || '', scopedVars), filters), + query: renderedQuery, bucketAggs: query.bucketAggs?.map(interpolateBucketAgg), }; @@ -353,13 +658,71 @@ export class BaseQuickwitDataSource } let finalQuery = query; adhocFilters.forEach((filter) => { - finalQuery = addAddHocFilter(finalQuery, filter); + const fieldType = this.fieldTypes[filter.key]; + const isText = fieldType === 'text'; + const value = filter.value ?? ''; + + // For text fields with simple token values, upgrade '=' to 'term' (and + // '!=' to 'not term') so the emitted query doesn't require positions. + let effective = filter; + if (isText && isSimpleToken(value)) { + if (filter.operator === '=') { + effective = { ...filter, operator: 'term' }; + } else if (filter.operator === '!=') { + effective = { ...filter, operator: 'not term' }; + } + } + + finalQuery = addAddHocFilter(finalQuery, effective); }); return finalQuery; } } -export function formatQuery(value: string | string[], variable: any): string { + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function collectFieldNames(value: unknown, fields: Set, prefix = '') { + if (Array.isArray(value)) { + value.forEach((item) => collectFieldNames(item, fields, prefix)); + return; + } + if (!isRecord(value)) { + if (prefix) { + fields.add(prefix); + } + return; + } + + Object.entries(value).forEach(([key, nested]) => { + const fieldName = prefix ? `${prefix}.${key}` : key; + collectFieldNames(nested, fields, fieldName); + }); +} + +function escapeRegExp(value: string) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function inferFieldNameFromQuery(queryString: string | undefined, variable: any) { + const names = [variable?.id, variable?.name].filter((name): name is string => typeof name === 'string' && name !== ''); + if (!queryString || names.length === 0) { + return undefined; + } + for (const name of names) { + const escapedName = escapeRegExp(name); + const pattern = new RegExp(`(?:^|[\\s(])([^\\s:()]+):(\\$\\{${escapedName}\\}|\\$${escapedName}|\\[\\[${escapedName}\\]\\])`); + const match = queryString.match(pattern); + if (match?.[1]) { + return match[1]; + } + } + return undefined; +} + +export function formatQuery(value: string | string[], variable: any, queryString?: string): string { if (typeof value === 'string') { return luceneEscape(value); } @@ -373,19 +736,17 @@ export function formatQuery(value: string | string[], variable: any): string { } catch (e) { fieldName = undefined; } + fieldName = typeof fieldName === 'string' ? fieldName : inferFieldNameFromQuery(queryString, variable); const quotedValues = value.map((val) => '"' + luceneEscape(val) + '"'); // Quickwit query language does not support fieldName:(value1 OR value2 OR....) // like lucene does. // When we know the fieldName, we can directly generate a query // fieldName:value1 OR fieldName:value2 OR ... - // But when we don't know the fieldName, the simplest is to generate a query - // with the IN operator. Unfortunately, IN operator does not work on JSON field. - // TODO: fix that by using doing a regex on queryString to find the fieldName. - // Note that variable.id gives the name of the template variable to interpolate, - // so if we have `fieldName:${variable.id}` in the queryString, we can isolate - // the fieldName. + // If the variable query does not carry a field, infer it from common + // `field:$var` query strings. Without a field, fall back to default-search + // clauses instead of a bare IN set, which is easy to misread as field-scoped. if (typeof fieldName !== 'string') { - return 'IN [' + quotedValues.join(' ') + ']'; + return quotedValues.join(' OR '); } return quotedValues.join(' OR ' + fieldName + ':'); } else { diff --git a/src/datasource/processResponse.test.ts b/src/datasource/processResponse.test.ts new file mode 100644 index 0000000..3843b58 --- /dev/null +++ b/src/datasource/processResponse.test.ts @@ -0,0 +1,233 @@ +import { DataFrame, Field, FieldType } from '@grafana/data'; +import { processLogsDataFrame } from './processResponse'; + +function makeField(name: string, type: FieldType, values: any[]): Field { + return { name, type, config: {}, values }; +} + +function makeDataFrame(fields: Field[], refId = 'A'): DataFrame { + return { + refId, + fields, + length: fields[0]?.values.length ?? 0, + }; +} + +function makeDatasource(overrides: { logMessageField?: string; dataLinks?: any[] } = {}) { + return { + logMessageField: overrides.logMessageField ?? '', + dataLinks: overrides.dataLinks ?? [], + } as any; +} + +describe('processLogsDataFrame', () => { + describe('with logMessageField configured', () => { + it('uses configured field value', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000, 2000]), + makeField('line', FieldType.string, ['hello world', 'goodbye world']), + makeField('level', FieldType.string, ['info', 'error']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values).toEqual(['hello world', 'goodbye world']); + }); + + it('joins multiple configured fields with key=value format', () => { + const ds = makeDatasource({ logMessageField: 'method,path,status' }); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('method', FieldType.string, ['GET']), + makeField('path', FieldType.string, ['/blog']), + makeField('status', FieldType.string, ['200']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('method=GET path=/blog status=200'); + }); + + it('falls back when configured field does not exist', () => { + const ds = makeDatasource({ logMessageField: 'nonexistent' }); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('body.message', FieldType.string, ['the real message']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('the real message'); + }); + + it('falls back when configured field value is empty for a row', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000, 2000]), + makeField('line', FieldType.string, ['has content', '']), + makeField('body.message', FieldType.string, ['', 'fallback message']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('has content'); + expect(df.fields[1].values[1]).toBe('fallback message'); + }); + }); + + describe('OTEL fallback (no logMessageField)', () => { + it('picks body.message when present', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('body.message', FieldType.string, ['GET /assets/app.js HTTP/1.1 200']), + makeField('body.stream', FieldType.string, ['stdout']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('GET /assets/app.js HTTP/1.1 200'); + }); + + it('picks attributes.message when body.message is absent', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('attributes.message', FieldType.string, ['SSO user already exists']), + makeField('attributes.severity', FieldType.string, ['INFO']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('SSO user already exists'); + }); + + it('prefers body.message over attributes.message', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('body.message', FieldType.string, ['from body']), + makeField('attributes.message', FieldType.string, ['from attributes']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('from body'); + }); + + it('builds key=value summary when no well-known fields exist', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('attributes.method', FieldType.string, ['GET']), + makeField('attributes.path', FieldType.string, ['/blog']), + makeField('attributes.status', FieldType.number, [200]), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('method=GET path=/blog status=200'); + }); + + it('strips attributes. prefix in key=value summary', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('attributes.controller', FieldType.string, ['BlogController']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].values[0]).toBe('controller=BlogController'); + }); + + it('skips metadata fields in key=value summary', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000]), + makeField('attributes.method', FieldType.string, ['GET']), + makeField('attributes.pod_name', FieldType.string, ['rx-production-abc123']), + makeField('attributes.node_labels.arch', FieldType.string, ['amd64']), + makeField('sort', FieldType.other, [[1684398201000]]), + makeField('severity_text', FieldType.string, ['INFO']), + makeField('body.stream', FieldType.string, ['stdout']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('method=GET'); + }); + + it('handles mixed log types per row', () => { + const ds = makeDatasource(); + const df = makeDataFrame([ + makeField('timestamp', FieldType.time, [1000, 2000, 3000]), + makeField('attributes.message', FieldType.string, ['SSO login', '', '']), + makeField('attributes.method', FieldType.string, ['', 'GET', '']), + makeField('attributes.path', FieldType.string, ['', '/blog', '']), + makeField('body.message', FieldType.string, ['', '', 'raw nginx log line']), + ]); + + processLogsDataFrame(ds, df); + + expect(df.fields[1].name).toBe('$qw_message'); + expect(df.fields[1].values[0]).toBe('SSO login'); + expect(df.fields[1].values[1]).toBe('method=GET path=/blog'); + expect(df.fields[1].values[2]).toBe('raw nginx log line'); + }); + }); + + describe('edge cases', () => { + it('skips empty dataframes', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df = makeDataFrame([]); + + processLogsDataFrame(ds, df); + + expect(df.fields.length).toBe(0); + }); + + it('skips log-volume dataframes', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df = makeDataFrame( + [ + makeField('timestamp', FieldType.time, [1000]), + makeField('line', FieldType.string, ['hello']), + ], + 'log-volume-A' + ); + + processLogsDataFrame(ds, df); + + const fieldNames = df.fields.map((f) => f.name); + expect(fieldNames).not.toContain('$qw_message'); + }); + + it('skips dataframes with no refId', () => { + const ds = makeDatasource({ logMessageField: 'line' }); + const df: DataFrame = { + refId: undefined, + fields: [ + makeField('timestamp', FieldType.time, [1000]), + makeField('line', FieldType.string, ['hello']), + ], + length: 1, + }; + + processLogsDataFrame(ds, df); + + const fieldNames = df.fields.map((f) => f.name); + expect(fieldNames).not.toContain('$qw_message'); + }); + }); +}); diff --git a/src/datasource/processResponse.ts b/src/datasource/processResponse.ts index 6743cbf..45e8fe8 100644 --- a/src/datasource/processResponse.ts +++ b/src/datasource/processResponse.ts @@ -17,6 +17,58 @@ export function getQueryResponseProcessor(datasource: BaseQuickwitDataSource, re }; } function getCustomFieldName(fieldname: string) { return `$qw_${fieldname}`; } + +const OTEL_MESSAGE_FIELDS = ['body.message', 'attributes.message']; + +const SKIP_FIELD_PREFIXES = [ + 'attributes.pod_', 'attributes.node_labels.', 'attributes.namespace_labels.', + 'attributes.container_image', 'attributes.pod_owner', +]; +const SKIP_FIELD_NAMES = new Set([ + 'sort', 'severity_text', 'body.stream', +]); + +function isMetadataField(name: string, timeField: string): boolean { + if (name === timeField || SKIP_FIELD_NAMES.has(name)) { + return true; + } + return SKIP_FIELD_PREFIXES.some((prefix) => name.startsWith(prefix)); +} + +function stripPrefix(name: string): string { + if (name.startsWith('attributes.')) { + return name.slice('attributes.'.length); + } + if (name.startsWith('body.')) { + return name.slice('body.'.length); + } + return name; +} + +function buildFallbackMessage(dataFrame: DataFrame, rowIdx: number, timeFieldName: string): string { + for (const candidate of OTEL_MESSAGE_FIELDS) { + const field = dataFrame.fields.find((f) => f.name === candidate); + if (field) { + const val = field.values[rowIdx]; + if (val != null && val !== '') { + return String(val); + } + } + } + + const parts: string[] = []; + for (const field of dataFrame.fields) { + if (isMetadataField(field.name, timeFieldName) || field.type === FieldType.time) { + continue; + } + const val = field.values[rowIdx]; + if (val != null && val !== '') { + parts.push(`${stripPrefix(field.name)}=${val}`); + } + } + return parts.join(' '); +} + export function processLogsDataFrame(datasource: BaseQuickwitDataSource, dataFrame: DataFrame) { // Ignore log volume dataframe, no need to add links or a displayed message field. if (!dataFrame.refId || dataFrame.refId.startsWith('log-volume')) { @@ -26,39 +78,47 @@ export function processLogsDataFrame(datasource: BaseQuickwitDataSource, dataFra if (dataFrame.length===0 || dataFrame.fields.length === 0) { return; } - if (datasource.logMessageField) { - const messageFields = datasource.logMessageField.split(','); - let field_idx_list = []; - for (const messageField of messageFields) { - const field_idx = dataFrame.fields.findIndex((field) => field.name === messageField); - if (field_idx !== -1) { - field_idx_list.push(field_idx); - } + + const configuredFields = datasource.logMessageField ? datasource.logMessageField.split(',') : []; + const field_idx_list: number[] = []; + for (const messageField of configuredFields) { + const field_idx = dataFrame.fields.findIndex((field) => field.name === messageField); + if (field_idx !== -1) { + field_idx_list.push(field_idx); } - const displayedMessages = Array(dataFrame.length); - for (let idx = 0; idx < dataFrame.length; idx++) { - let displayedMessage = ""; - // If we have only one field, we assume the field name is obvious for the user and we don't need to show it. - if (field_idx_list.length === 1) { - displayedMessage = `${dataFrame.fields[field_idx_list[0]].values[idx]}`; - } else { - for (const field_idx of field_idx_list) { - displayedMessage += ` ${dataFrame.fields[field_idx].name}=${dataFrame.fields[field_idx].values[idx]}`; - } + } + + const timeFieldName = dataFrame.fields.find((f) => f.type === FieldType.time)?.name ?? ''; + const displayedMessages = Array(dataFrame.length); + + for (let idx = 0; idx < dataFrame.length; idx++) { + let displayedMessage = ""; + + if (field_idx_list.length === 1) { + displayedMessage = `${dataFrame.fields[field_idx_list[0]].values[idx] ?? ''}`; + } else if (field_idx_list.length > 1) { + for (const field_idx of field_idx_list) { + displayedMessage += ` ${dataFrame.fields[field_idx].name}=${dataFrame.fields[field_idx].values[idx]}`; } - displayedMessages[idx] = displayedMessage.trim(); + displayedMessage = displayedMessage.trim(); } - const newField: Field = { - name: getCustomFieldName('message'), - type: FieldType.string, - config: {}, - values: displayedMessages, - }; - const [timestamp, ...rest] = dataFrame.fields; - dataFrame.fields = [timestamp, newField, ...rest]; + if (!displayedMessage) { + displayedMessage = buildFallbackMessage(dataFrame, idx, timeFieldName); + } + + displayedMessages[idx] = displayedMessage; } + const newField: Field = { + name: getCustomFieldName('message'), + type: FieldType.string, + config: {}, + values: displayedMessages, + }; + const [timestamp, ...rest] = dataFrame.fields; + dataFrame.fields = [timestamp, newField, ...rest]; + if (!datasource.dataLinks.length) { return; } diff --git a/src/datasource/supplementaryQueries.ts b/src/datasource/supplementaryQueries.ts index 582d069..bf4f63d 100644 --- a/src/datasource/supplementaryQueries.ts +++ b/src/datasource/supplementaryQueries.ts @@ -102,6 +102,7 @@ export function withSupplementaryQueries { query: '', metrics: [defaultMetricAgg()], bucketAggs: [defaultBucketAgg()], + filters: [] }; const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); diff --git a/src/hooks/useFields.ts b/src/hooks/useFields.ts index 400e071..06579ce 100644 --- a/src/hooks/useFields.ts +++ b/src/hooks/useFields.ts @@ -6,6 +6,7 @@ import { isBucketAggregationType } from '../components/QueryEditor/BucketAggrega import { useDatasource, useRange } from '../components/QueryEditor/ElasticsearchQueryContext'; import { isMetricAggregationType } from '../components/QueryEditor/MetricAggregationsEditor/aggregations'; import { MetricAggregationType, BucketAggregationType } from '../types'; +import { fuzzySearchMatch, fuzzySearchSort } from '@/utils'; type AggregationType = BucketAggregationType | MetricAggregationType; @@ -46,14 +47,17 @@ const toSelectableValue = ({ text }: MetricFindValue): SelectableValue = value: text, }); +type MatchType = 'contains' | 'startsWith' | 'fuzzy' + /** * Returns a function to query the configured datasource for autocomplete values for the specified aggregation type or data types. * Each aggregation can be run on different types, for example avg only operates on numeric fields, geohash_grid only on geo_point fields. * If an aggregation type is provided, the promise will resolve with all fields suitable to be used as a field for the given aggregation. * If an array of types is providem the promise will resolve with all the fields matching the provided types. - * @param aggregationType the type of aggregation to get fields for + * @param type the type of aggregation to get fields for + * @param matchType the type of matching to use when filtering fields based on the query string. Defaults to 'contains'. */ -export const useFields = (type: AggregationType | string[]) => { +export const useFields = (type: AggregationType | string[], matchType: MatchType = 'contains') => { const datasource = useDatasource(); const range = useRange(); const filter = Array.isArray(type) ? type : getFilter(type); @@ -65,6 +69,18 @@ export const useFields = (type: AggregationType | string[]) => { rawFields = await lastValueFrom(datasource.getFields({aggregatable:true, type:filter, range:range})); } - return rawFields.filter(({ text }) => q === undefined || text.includes(q)).map(toSelectableValue); + const fields = rawFields + .filter(({ text }) => { + if (q === undefined) { + return true; + } + if (matchType === 'fuzzy') { + return fuzzySearchMatch(text, q); + } + return matchType === 'contains' ? text.includes(q) : text.startsWith(q) + }); + + return (matchType === 'fuzzy' ? fuzzySearchSort(fields, ({ text }) => text, q) : fields) + .map(toSelectableValue); }; }; diff --git a/src/hooks/useNextId.test.tsx b/src/hooks/useNextId.test.tsx index 1b0d287..0e76dcb 100644 --- a/src/hooks/useNextId.test.tsx +++ b/src/hooks/useNextId.test.tsx @@ -16,6 +16,7 @@ describe('useNextId', () => { query: '', metrics: [{ id: '1', type: 'avg' }], bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; const wrapper = ({ children }: PropsWithChildren<{}>) => { return ( diff --git a/src/modifyQuery.test.ts b/src/modifyQuery.test.ts new file mode 100644 index 0000000..aa4772d --- /dev/null +++ b/src/modifyQuery.test.ts @@ -0,0 +1,297 @@ +import { addAddHocFilter } from './modifyQuery'; + +describe('addAddHocFilter', () => { + describe('array values', () => { + it('unwraps single-element array into a phrase query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["paperclip"]', + }); + expect(result).toBe('attributes.tags:"paperclip"'); + }); + + it('unwraps multi-element array into IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["paperclip","stapler"]', + }); + expect(result).toBe('attributes.tags:IN ["paperclip" "stapler"]'); + }); + + it('negated single-element array produces negated phrase query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '!=', + value: '["paperclip"]', + }); + expect(result).toBe('-attributes.tags:"paperclip"'); + }); + + it('negated multi-element array produces negated IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '!=', + value: '["paperclip","stapler"]', + }); + expect(result).toBe('-attributes.tags:IN ["paperclip" "stapler"]'); + }); + + it('appends array filter to existing query with AND', () => { + const result = addAddHocFilter('status:200', { + key: 'attributes.tags', + operator: '=', + value: '["paperclip"]', + }); + expect(result).toBe('status:200 AND attributes.tags:"paperclip"'); + }); + + it('handles single-element array with spaces in value', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["foo bar"]', + }); + expect(result).toBe('attributes.tags:"foo bar"'); + }); + + it('handles single-element array with colons in value', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["foo:bar"]', + }); + expect(result).toBe('attributes.tags:"foo:bar"'); + }); + + it('handles multi-element array with spaces in values', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["foo bar","baz qux"]', + }); + expect(result).toBe('attributes.tags:IN ["foo bar" "baz qux"]'); + }); + + it('handles single-element numeric arrays', () => { + const result = addAddHocFilter('', { + key: 'attributes.codes', + operator: '=', + value: '[200]', + }); + expect(result).toBe('attributes.codes:200'); + }); + + it('handles multi-element numeric arrays with IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.codes', + operator: '=', + value: '[200,500]', + }); + expect(result).toBe('attributes.codes:IN [200 500]'); + }); + + it('keeps negative numeric array values unquoted', () => { + const result = addAddHocFilter('', { + key: 'attributes.deltas', + operator: '=', + value: '[-1,2]', + }); + expect(result).toBe('attributes.deltas:IN [-1 2]'); + }); + + it('handles boolean arrays with IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.flags', + operator: '=', + value: '[true,false]', + }); + expect(result).toBe('attributes.flags:IN [true false]'); + }); + + it('handles mixed scalar arrays with IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.values', + operator: '=', + value: '["paperclip",200,true]', + }); + expect(result).toBe('attributes.values:IN ["paperclip" 200 true]'); + }); + + it('negates numeric arrays with IN set query', () => { + const result = addAddHocFilter('', { + key: 'attributes.codes', + operator: '!=', + value: '[200,500]', + }); + expect(result).toBe('-attributes.codes:IN [200 500]'); + }); + + it('handles array values containing double quotes', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: '=', + value: '["say \\"hello\\""]', + }); + expect(result).toBe('attributes.tags:"say \\"hello\\""'); + }); + + it('passes through non-array bracket strings unchanged', () => { + const result = addAddHocFilter('', { + key: 'attributes.message', + operator: '=', + value: '[not json', + }); + expect(result).toBe('attributes.message:"[not json"'); + }); + + it('term operator still produces unquoted query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: 'term', + value: 'paperclip', + }); + expect(result).toBe('attributes.tags:paperclip'); + }); + + it('not term operator still produces negated unquoted query', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: 'not term', + value: 'paperclip', + }); + expect(result).toBe('-attributes.tags:paperclip'); + }); + }); + + describe('scalar value filters', () => { + it('equality on simple string value', () => { + const result = addAddHocFilter('', { + key: 'attributes.controller', + operator: '=', + value: 'BlogController', + }); + expect(result).toBe('attributes.controller:"BlogController"'); + }); + + it('appends to existing query with AND', () => { + const result = addAddHocFilter('status:200', { + key: 'attributes.controller', + operator: '=', + value: 'BlogController', + }); + expect(result).toBe('status:200 AND attributes.controller:"BlogController"'); + }); + + it('renders numeric equality filters as unquoted literals', () => { + const result = addAddHocFilter('', { + key: 'attributes.status_code', + operator: '=', + value: '200', + }); + expect(result).toBe('attributes.status_code:200'); + }); + + it('keeps numeric zero as a valid filter value', () => { + const result = addAddHocFilter('', { + key: 'attributes.retry_count', + operator: '=', + value: 0 as any, + }); + expect(result).toBe('attributes.retry_count:0'); + }); + + it('keeps boolean false as a valid filter value', () => { + const result = addAddHocFilter('', { + key: 'attributes.cache_hit', + operator: '=', + value: false as any, + }); + expect(result).toBe('attributes.cache_hit:false'); + }); + + it('renders negated boolean equality filters as unquoted literals', () => { + const result = addAddHocFilter('', { + key: 'attributes.cache_hit', + operator: '!=', + value: false as any, + }); + expect(result).toBe('-attributes.cache_hit:false'); + }); + + it('exists operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: 'exists', + value: '', + }); + expect(result).toBe('attributes.tags:*'); + }); + + it('not exists operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.tags', + operator: 'not exists', + value: '', + }); + expect(result).toBe('-attributes.tags:*'); + }); + + it('regex operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.controller', + operator: '=~', + value: 'Blog.*', + }); + expect(result).toBe('attributes.controller:/Blog.*/'); + }); + + it('greater than operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.duration', + operator: '>', + value: '100', + }); + expect(result).toBe('attributes.duration:>100'); + }); + + it('less than operator', () => { + const result = addAddHocFilter('', { + key: 'attributes.duration', + operator: '<', + value: '100', + }); + expect(result).toBe('attributes.duration:<100'); + }); + }); + + describe('edge cases', () => { + it('returns query unchanged when key is empty', () => { + const result = addAddHocFilter('existing', { + key: '', + operator: '=', + value: 'test', + }); + expect(result).toBe('existing'); + }); + + it('returns query unchanged when value is empty for non-exists operators', () => { + const result = addAddHocFilter('existing', { + key: 'field', + operator: '=', + value: '', + }); + expect(result).toBe('existing'); + }); + + it('treats empty JSON array as no-op', () => { + const result = addAddHocFilter('existing', { + key: 'attributes.tags', + operator: '=', + value: '[]', + }); + expect(result).toBe('existing'); + }); + }); +}); diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index 8bd58b4..5fed8b7 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -1,11 +1,48 @@ import { escapeFilter, escapeFilterValue, concatenate, LuceneQuery } from 'utils/lucene'; import { AdHocVariableFilter } from '@grafana/data'; +type FilterArrayElement = string | number | boolean; + +function isFilterArrayElement(value: unknown): value is FilterArrayElement { + return ['string', 'number', 'boolean'].includes(typeof value); +} + +function tryParseJsonArray(value: string): FilterArrayElement[] | null { + if (!value.trimStart().startsWith('[')) { + return null; + } + try { + const parsed = JSON.parse(value); + if (Array.isArray(parsed) && parsed.every(isFilterArrayElement)) { + return parsed; + } + } catch { + // not valid JSON + } + return null; +} + +function formatArrayElement(value: FilterArrayElement) { + if (typeof value === 'string') { + return `"${escapeFilterValue(value)}"`; + } + return String(value); +} + +function hasFilterValue(value: unknown) { + return value !== undefined && value !== null && String(value) !== ''; +} + +function isLiteralValue(value: string) { + return /^-?(?:\d+(?:\.\d+)?|\.\d+)(?:e[+-]?\d+)?$/i.test(value) || value === 'true' || value === 'false'; +} + /** * Adds a label:"value" expression to the query. */ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): string { - if (!filter.key || !filter.value) { + const hasValidValue = ['exists', 'not exists'].includes(filter.operator) || hasFilterValue(filter.value) + if (!filter.key || !hasValidValue) { return query; } @@ -17,6 +54,27 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str const equalityFilters = ['=', '!=']; if (equalityFilters.includes(filter.operator)) { + const modifier = filter.operator === '=' ? '' : '-'; + const key = escapeFilter(filter.key); + // Grafana stringifies array values (e.g. ["paperclip","stapler"]) before + // passing them as filter values. Tantivy indexes array elements as + // individual terms — there's no way to match on array length, order, or + // exact composition. For multi-element arrays we use IN (match any), + // which is the most useful behavior for log exploration filters. + const arrayElements = tryParseJsonArray(filter.value); + if (arrayElements !== null) { + if (arrayElements.length === 0) { + return query; + } + if (arrayElements.length === 1) { + return concatenate(query, `${modifier}${key}:${formatArrayElement(arrayElements[0])}`, 'AND'); + } + const terms = arrayElements.map(formatArrayElement).join(' '); + return concatenate(query, `${modifier}${key}:IN [${terms}]`, 'AND'); + } + if (isLiteralValue(filter.value)) { + return concatenate(query, `${modifier}${key}:${filter.value}`, 'AND'); + } return LuceneQuery.parse(query).addFilter(filter.key, filter.value, filter.operator === '=' ? '' : '-').toString(); } /** @@ -25,6 +83,7 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str */ const key = escapeFilter(filter.key); const value = escapeFilterValue(filter.value); + const termValue = escapeFilter(filter.value); let addHocFilter = ''; switch (filter.operator) { case '=~': @@ -39,6 +98,18 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str case '<': addHocFilter = `${key}:<${value}`; break; + case 'term': + addHocFilter = `${key}:${termValue}`; + break; + case 'not term': + addHocFilter = `-${key}:${termValue}`; + break; + case 'exists': + addHocFilter = `${key}:*`; + break; + case 'not exists': + addHocFilter = `-${key}:*`; + break; } return concatenate(query, addHocFilter,'AND'); } diff --git a/src/queryDef.ts b/src/queryDef.ts index 9fa54bb..0b7cc0e 100644 --- a/src/queryDef.ts +++ b/src/queryDef.ts @@ -6,7 +6,9 @@ import { MovingAverageModelOption, MetricAggregationType, DateHistogram, + QueryFilter, } from './types'; +import { newFilterId } from '@/utils/uid'; export const extendedStats: ExtendedStat[] = [ { label: 'Avg', value: 'avg' }, @@ -35,6 +37,53 @@ export function defaultBucketAgg(id = '1'): DateHistogram { return { type: 'date_histogram', id, settings: { interval: 'auto' } }; } +export type FilterFieldCategory = 'text' | 'keyword' | 'number' | 'date' | 'boolean' | 'other'; + +export interface FilterOperation { + label: string; + value: string; + // Field categories this operation applies to. Undefined = always available. + types?: FilterFieldCategory[]; +} + +export const filterOperations: FilterOperation[] = [ + { label: 'is', value: '=' }, + { label: 'is not', value: '!=' }, + { label: 'contains', value: 'term', types: ['text'] }, + { label: 'does not contain', value: 'not term', types: ['text'] }, + { label: '>', value: '>', types: ['number', 'date'] }, + { label: '<', value: '<', types: ['number', 'date'] }, + { label: 'exists', value: 'exists' }, + { label: 'does not exist', value: 'not exists' }, +]; + +/** + * Map a Quickwit/Elasticsearch raw field type (as returned by _field_caps) to + * a coarser category used to decide which operators to expose in the filter UI. + */ +export function categorizeFieldType(rawType?: string): FilterFieldCategory { + if (!rawType) {return 'other';} + if (rawType === 'text') {return 'text';} + if (rawType === 'keyword') {return 'keyword';} + if (['long', 'integer', 'int', 'short', 'byte', 'double', 'float', 'scaled_float', 'half_float', 'unsigned_long'].includes(rawType)) {return 'number';} + if (['date', 'date_nanos'].includes(rawType)) {return 'date';} + if (rawType === 'boolean' || rawType === 'bool') {return 'boolean';} + return 'other'; +} + +/** + * Operations available for a given field category. When the category is 'other' + * (unknown field) we return everything so the user still has full control. + */ +export function filterOperationsFor(category: FilterFieldCategory): FilterOperation[] { + if (category === 'other') {return filterOperations;} + return filterOperations.filter((op) => !op.types || op.types.includes(category)); +} + +export function defaultFilter(id = newFilterId()): QueryFilter { + return { id, filter: { key: '', operator: filterOperations[0].value, value: '' } }; +} + export const findMetricById = (metrics: MetricAggregation[], id: MetricAggregation['id']) => metrics.find((metric) => metric.id === id); diff --git a/src/quickwit.ts b/src/quickwit.ts index 2d1599f..8c38aa9 100644 --- a/src/quickwit.ts +++ b/src/quickwit.ts @@ -2,6 +2,8 @@ import { DataSourceJsonData } from "@grafana/data"; import { DataLinkConfig } from "./types"; import { DefaultsConfigOverrides } from "store/defaults/conf"; +export type FilterAutocompleteChainMode = 'none' | 'sample' | 'full'; + export interface QuickwitOptions extends DataSourceJsonData { timeField: string; interval?: string; @@ -9,6 +11,10 @@ export interface QuickwitOptions extends DataSourceJsonData { logLevelField?: string; dataLinks?: DataLinkConfig[]; index: string; + filterAutocompleteLimit?: string; + filterAutocompleteChainMode?: FilterAutocompleteChainMode; + // Backward compatibility for configs created before the mode selector. + filterAutocompleteUseFilterChains?: boolean; queryEditorConfig?: { defaults?: DefaultsConfigOverrides } diff --git a/src/utils/index.test.ts b/src/utils/index.test.ts index 1bcec2c..4552623 100644 --- a/src/utils/index.test.ts +++ b/src/utils/index.test.ts @@ -1,4 +1,4 @@ -import { extractJsonPayload } from "utils"; +import { extractJsonPayload, fuzzySearchMatch, fuzzySearchSort, isSimpleToken } from "utils"; describe('Test utils.extractJsonPayload', () => { it('Extract valid JSON', () => { @@ -18,3 +18,37 @@ describe('Test utils.extractJsonPayload', () => { expect(result).toEqual(null); }); }); + +describe('fuzzy search helpers', () => { + it('matches field names using token abbreviations', () => { + expect(fuzzySearchMatch('attributes.grpc_message', 'grpc msg')).toBe(true); + }); + + it('matches values with missing characters', () => { + expect(fuzzySearchMatch('Error invalid token', 'err invld')).toBe(true); + }); + + it('sorts stronger matches first', () => { + const result = fuzzySearchSort( + ['body.message', 'attributes.grpc_message', 'attributes.http_status'], + (value) => value, + 'grpc msg' + ); + + expect(result[0]).toBe('attributes.grpc_message'); + }); + + it('does not match unrelated values', () => { + expect(fuzzySearchMatch('attributes.grpc_message', 'status code')).toBe(false); + }); +}); + +describe('isSimpleToken', () => { + it('accepts analyzer-friendly bare tokens', () => { + expect(isSimpleToken('auth_api_1')).toBe(true); + }); + + it('rejects punctuated values that should be quoted as phrases', () => { + expect(isSimpleToken('auth-api')).toBe(false); + }); +}); diff --git a/src/utils/index.ts b/src/utils/index.ts index 8e5a826..ace2cb4 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -131,3 +131,132 @@ export const fieldTypeMap: Record = { float: 'number', scaled_float: 'number' }; + +export const isSet = (v: string) => v !== '' && v !== undefined && v !== null; + +export const hasWhiteSpace = (s: string) => /\s/g.test(s); + +export const isSimpleToken = (s: string) => /^[A-Za-z0-9_]+$/.test(s); + +const normalizeSearchText = (value: string) => value.toLowerCase(); + +const searchTokens = (value: string) => + normalizeSearchText(value) + .split(/[^a-z0-9]+/g) + .filter(Boolean); + +const isSubsequence = (needle: string, haystack: string) => { + let needleIndex = 0; + for (let haystackIndex = 0; haystackIndex < haystack.length && needleIndex < needle.length; haystackIndex++) { + if (needle[needleIndex] === haystack[haystackIndex]) { + needleIndex++; + } + } + return needleIndex === needle.length; +}; + +const levenshteinDistance = (a: string, b: string, maxDistance: number) => { + if (Math.abs(a.length - b.length) > maxDistance) { + return maxDistance + 1; + } + + let previous = Array.from({ length: b.length + 1 }, (_, index) => index); + for (let i = 1; i <= a.length; i++) { + const current = [i]; + let rowMin = current[0]; + for (let j = 1; j <= b.length; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + current[j] = Math.min( + current[j - 1] + 1, + previous[j] + 1, + previous[j - 1] + cost + ); + rowMin = Math.min(rowMin, current[j]); + } + if (rowMin > maxDistance) { + return maxDistance + 1; + } + previous = current; + } + + return previous[b.length]; +}; + +const fuzzyTokenScore = (candidateTokens: string[], compactCandidate: string, token: string): number | null => { + if (!token) { + return 0; + } + + let best: number | null = null; + for (const candidateToken of candidateTokens) { + let score: number | null = null; + if (candidateToken === token) { + score = 0; + } else if (candidateToken.startsWith(token)) { + score = 10 + candidateToken.length - token.length; + } else if (candidateToken.includes(token)) { + score = 25 + candidateToken.indexOf(token); + } else if (isSubsequence(token, candidateToken)) { + score = 50 + candidateToken.length - token.length; + } else if (token.length >= 3) { + const maxDistance = Math.max(1, Math.floor(token.length / 3)); + const distance = levenshteinDistance(token, candidateToken, maxDistance); + if (distance <= maxDistance) { + score = 80 + distance * 10 + Math.abs(candidateToken.length - token.length); + } + } + + if (score !== null && (best === null || score < best)) { + best = score; + } + } + + if (best === null && compactCandidate.includes(token)) { + return 35 + compactCandidate.indexOf(token); + } + + return best; +}; + +export const fuzzySearchScore = (text: string, query?: string): number | null => { + const trimmedQuery = query?.trim(); + if (!trimmedQuery) { + return 0; + } + + const normalizedText = normalizeSearchText(text); + const normalizedQuery = normalizeSearchText(trimmedQuery); + if (normalizedText === normalizedQuery) { + return 0; + } + if (normalizedText.startsWith(normalizedQuery)) { + return 5 + normalizedText.length - normalizedQuery.length; + } + if (normalizedText.includes(normalizedQuery)) { + return 15 + normalizedText.indexOf(normalizedQuery); + } + + const candidateTokens = searchTokens(text); + const compactCandidate = candidateTokens.join(''); + const queryTokens = searchTokens(trimmedQuery); + let total = 0; + for (const token of queryTokens) { + const score = fuzzyTokenScore(candidateTokens, compactCandidate, token); + if (score === null) { + return null; + } + total += score; + } + + return total; +}; + +export const fuzzySearchMatch = (text: string, query?: string) => fuzzySearchScore(text, query) !== null; + +export function fuzzySearchSort(items: T[], getText: (item: T) => string, query?: string): T[] { + return items + .map((item, index) => ({ item, index, score: fuzzySearchScore(getText(item), query) })) + .filter((item): item is { item: T; index: number; score: number } => item.score !== null) + .sort((a, b) => a.score - b.score || a.index - b.index) + .map(({ item }) => item); +} diff --git a/src/utils/uid.ts b/src/utils/uid.ts index bf51aac..9f029c7 100644 --- a/src/utils/uid.ts +++ b/src/utils/uid.ts @@ -1,3 +1,5 @@ +import { QueryFilter } from '@/dataquery.gen'; + export function uidMaker(prefix: string){ let i = 1; return { @@ -10,3 +12,7 @@ export function uidMaker(prefix: string){ } } } + +export function newFilterId(): QueryFilter['id'] { + return Math.floor(Math.random() * 100_000_000).toString() +} diff --git a/tests/quickwit.spec.ts b/tests/quickwit.spec.ts index a2fa64d..f7c0fb7 100644 --- a/tests/quickwit.spec.ts +++ b/tests/quickwit.spec.ts @@ -1,4 +1,5 @@ import { test, expect } from '@grafana/plugin-e2e'; +import type { Page } from '@playwright/test'; // Grafana (inside Docker) reaches Quickwit via the docker service name. // The test process (on the host) reaches the same Quickwit via localhost port mapping. @@ -20,17 +21,22 @@ test.afterEach(async () => { test('create datasource and explore logs', async ({ page }) => { let datasourceUid: string; + const quickwitDatasourceButton = page.locator('button').filter({ hasText: /^Quickwit$/ }).first(); await test.step('look for the plugin in the list of datasources', async () => { await page.goto('/connections/datasources/new'); + await dismissWhatsNewModal(page); await page.getByPlaceholder('Filter by name or type').fill('quickwit'); - await expect(page.getByText('Quickwit', { exact: true })).toBeVisible(); + await expect(quickwitDatasourceButton).toBeVisible(); }); await test.step('create datasource via UI', async () => { await page.goto('/connections/datasources/new'); + await dismissWhatsNewModal(page); await page.getByPlaceholder('Filter by name or type').fill('quickwit'); - await page.getByText('Quickwit', { exact: true }).click(); + // Grafana 13 sometimes leaves a portal overlay mounted above the picker. + // Force the click once the correct button is resolved. + await quickwitDatasourceButton.click({ force: true }); // Wait for the config form to load await expect(page.getByText('Index settings')).toBeVisible(); @@ -106,3 +112,15 @@ async function ingestDummyLogs(runId: string) { throw new Error('Ingest failed: no searchable logs'); } } + +async function dismissWhatsNewModal(page: Page) { + const dialog = page.getByRole('dialog', { name: "What's new in Grafana" }); + const closeButton = dialog.getByRole('button', { name: 'Close' }); + + await closeButton.waitFor({ state: 'visible', timeout: 3000 }).catch(() => {}); + + if (await dialog.isVisible().catch(() => false)) { + await closeButton.click({ force: true }); + await expect(dialog).toBeHidden({ timeout: 10000 }); + } +}