Skip to content

Commit

Permalink
uql updates
Browse files Browse the repository at this point in the history
  • Loading branch information
yesoreyeram committed Dec 27, 2021
1 parent ca0bb68 commit 5b11aca
Show file tree
Hide file tree
Showing 20 changed files with 629 additions and 140 deletions.
10 changes: 10 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@
"gomodcache",
"grafana",
"jsonpath",
"kusto",
"magefile",
"mainbg",
"mkdir",
"newyork",
"nodemodules",
"notcontains",
"notequals",
Expand All @@ -23,10 +25,18 @@
"plugincheck",
"scroller",
"starswith",
"strcat",
"stretchr",
"subnav",
"timeseries",
"todatetime",
"tolong",
"tolower",
"tonumber",
"topnav",
"tostring",
"toupper",
"unixtime",
"unstyled",
"visualisation",
"vlookup",
Expand Down
8 changes: 4 additions & 4 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ Change history of the project. All the feature updates, bug fixes, breaking chan
## [ 0.8.0 ] - unreleased

- custom query type for tsv files
- support for Node Graph
- moved docs website, try into their own folders
- E2E tests for config editor (placeholder) added
- support for UQL queries
- secure query params passed to all requests by default
- support for Node Graph panel
- basic E2E tests for config editor added
- typescript updates
- "as-is" data format added for debugging
- comma in the numbers are now ignored and considered as number
- variable editor, global query editor bug fixes. Previously, unable to add columns in variable editor
- support for UQL queries
- fixes [#191](https://github.com/yesoreyeram/grafana-infinity-datasource/issues/191), [#146](https://github.com/yesoreyeram/grafana-infinity-datasource/issues/146), [#210](https://github.com/yesoreyeram/grafana-infinity-datasource/issues/210)

## [ 0.7.10 ]
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "grafana-infinity-datasource",
"version": "0.8.0-dev.11",
"version": "0.8.0-beta.1",
"description": "JSON, CSV, XML, GraphQL & HTML datasource for Grafana. Do infinite things with Grafana.",
"main": "dist/module.js",
"scripts": {
Expand Down Expand Up @@ -53,7 +53,7 @@
"react": "16.12.0",
"react-dom": "16.12.0",
"rxjs": "6.6.3",
"uql": "0.0.4-dev.2",
"uql": "0.0.4",
"xml2js": "^0.4.23"
},
"devDependencies": {
Expand Down
3 changes: 3 additions & 0 deletions pkg/infinity/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ func GetQueryURL(settings InfinitySettings, query Query) (string, error) {
value := replaceSecret(param.Value, settings)
q.Set(param.Key, value)
}
for key, value := range settings.SecureQueryFields {
q.Set(key, value)
}
u.RawQuery = q.Encode()
return u.String(), nil
}
Expand Down
13 changes: 13 additions & 0 deletions pkg/infinity/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,19 @@ func Test_getQueryURL(t *testing.T) {
},
want: "https://foo.com/hello?foo=bar&key=val&key_one=val_one&key_two=val_two",
},
{
settings: infinity.InfinitySettings{
URL: "https://foo.com",
SecureQueryFields: map[string]string{
"key_one": "val_one",
"key_two": "val_two",
},
},
query: infinity.Query{
URL: "/hello?key=val&foo=bar&key_one=foo",
},
want: "https://foo.com/hello?foo=bar&key=val&key_one=val_one&key_two=val_two",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
134 changes: 134 additions & 0 deletions src/app/UQLProvider.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
import { DataFrame, FieldType, MutableDataFrame } from '@grafana/data';
import { applyUQL } from './UQLProvider';

describe('UQLProvider', () => {
it('basic array', async () => {
const result = await applyUQL(`parse-json | count`, `[{},{}]`, 'table', 'A');
const expected = { name: 'A', length: 1, fields: [{ name: 'result', type: FieldType.number, values: [2] }] };
expect(result).toStrictEqual(new MutableDataFrame(expected));
});
it('basic array with project', async () => {
const result = await applyUQL(
`parse-json
| project "full name"=strcat("name.first",' ',"name.last")
| extend "full name"=toupper("full name"), "full name"=trim("full name")
| order by "full name" asc`,
`[
{ "name": { "first" :"john", "last" : "doe"} },
{ "name": { "first" :" alice", "last" : "bob "} }
]`,
'table',
'A'
);
expect(result.length).toStrictEqual(2);
expect((result as DataFrame).fields.length).toStrictEqual(1);
expect((result as DataFrame).fields[0].name).toStrictEqual('full name');
expect((result as DataFrame).fields[0].values.toArray()).toStrictEqual(['ALICE BOB', 'JOHN DOE']);
});
it('basic array with summarize', async () => {
const result = await applyUQL(
`parse-json
| summarize "number of cities"=count(), "total population"=sum("population") by "country"
| extend "country"=toupper("country")
| order by "total population" desc`,
`[
{ "city": "tokyo", "country": "japan", "population": 200 },
{ "city": "newyork", "country": "usa", "population": 60 },
{ "city": "oslo", "country": "usa", "population": 40 },
{ "city": "new delhi", "country": "india", "population": 180 },
{ "city": "mumbai", "country": "india", "population": 150 }
]`,
'table',
'A'
);
expect(result.length).toStrictEqual(3);
expect((result as DataFrame).fields.length).toStrictEqual(3);
expect((result as DataFrame).fields[0].name).toStrictEqual('country');
expect((result as DataFrame).fields[0].values.toArray()).toStrictEqual(['INDIA', 'JAPAN', 'USA']);
expect((result as DataFrame).fields[1].name).toStrictEqual('number of cities');
expect((result as DataFrame).fields[1].values.toArray()).toStrictEqual([2, 1, 2]);
expect((result as DataFrame).fields[2].name).toStrictEqual('total population');
expect((result as DataFrame).fields[2].values.toArray()).toStrictEqual([330, 200, 100]);
});
it('numeric ops', async () => {
const result = await applyUQL(
`parse-json | project "a", "triple"=sum("a","a","a"),"thrice"=mul("a",3), sum("a","b"), diff("a","b"), mul("a","b")`,
`[ { "a": 12, "b" : 20 }, { "a" : 6, "b": 32} ]`,
'table',
'A'
);
expect(result.length).toStrictEqual(2);
expect((result as DataFrame).fields.length).toStrictEqual(6);
expect((result as DataFrame).fields[0].values.toArray()).toStrictEqual([12, 6]);
expect((result as DataFrame).fields[1].values.toArray()).toStrictEqual([36, 18]);
expect((result as DataFrame).fields[2].values.toArray()).toStrictEqual([36, 18]);
expect((result as DataFrame).fields[3].values.toArray()).toStrictEqual([32, 38]);
expect((result as DataFrame).fields[4].values.toArray()).toStrictEqual([-8, -26]);
expect((result as DataFrame).fields[5].values.toArray()).toStrictEqual([240, 192]);
});
it('csv', async () => {
const result = await applyUQL(
`parse-csv
| extend "population"=tolong("population")
| summarize "density"=sum("population") by "country"
| order by "density" desc
| limit 2`,
`country,population
india,300
india,1230
china,500
usa,
canada,200`,
'table',
'A'
);
expect(result.length).toStrictEqual(2);
expect((result as DataFrame).fields.length).toStrictEqual(2);
expect((result as DataFrame).fields[0].name).toStrictEqual('country');
expect((result as DataFrame).fields[0].values.toArray()).toStrictEqual(['india', 'china']);
expect((result as DataFrame).fields[1].name).toStrictEqual('density');
expect((result as DataFrame).fields[1].values.toArray()).toStrictEqual([1530, 500]);
});
it('xml', async () => {
const result = await applyUQL(
`parse-xml
| scope "rss.channel.item"
| extend "pubDate"=todatetime("pubDate")`,
`<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title><![CDATA[Amazon Web Services Service Status]]></title>
<link>http://status.aws.amazon.com/</link>
<language>en-us</language>
<lastBuildDate>Mon, 27 Dec 2021 00:05:42 PST</lastBuildDate>
<generator>AWS Service Health Dashboard RSS Generator</generator>
<description><![CDATA[Amazon Web Services Service Status]]></description>
<ttl>5</ttl>
<!-- You seem to care about knowing about your events, why not check out https://docs.aws.amazon.com/health/latest/ug/getting-started-api.html -->
<item>
<title><![CDATA[Informational message: [RESOLVED] Internet connectivity]]></title>
<link>http://status.aws.amazon.com/</link>
<pubDate>Fri, 24 Dec 2021 10:41:17 PST</pubDate>
<guid isPermaLink="false">http://status.aws.amazon.com/#internetconnectivity-ap-south-1_1640371277</guid>
<description><![CDATA[Between 8:59 AM and 9:32 AM PST and between 9:40 AM and 10:16 AM PST we observed Internet connectivity issues with a network provider outside of our network in the AP-SOUTH-1 Region. This impacted Internet connectivity from some customer networks to the AP-SOUTH-1 Region. Connectivity between EC2 instances and other AWS services within the Region was not impacted by this event. The issue has been resolved and we continue to work with the external provider to ensure it does not reoccur.
]]></description>
</item>
<item>
<title><![CDATA[Service is operating normally: [RESOLVED] Increased Error Rates with Directory Services AD Connector or Managed AD]]></title>
<link>http://status.aws.amazon.com/</link>
<pubDate>Wed, 22 Dec 2021 17:57:40 PST</pubDate>
<guid isPermaLink="false">http://status.aws.amazon.com/#directoryservice-us-east-1_1640224660</guid>
<description><![CDATA[Between 4:09 AM and 5:00 PM PST we experienced increased error rates for some customers using Directory Services AD Connector or Managed AD with Directory Services in US-EAST-1 Region. This also impacted some services, like Amazon WorkSpaces, that can be configured to use Directory Services for user authentication. The issue has been resolved and the service is operating normally. Customers using other Active Directory functionality were not impacted by this issue. If you experience any issues with this service or need further assistance, please contact AWS Support.]]></description>
</item>
</channel>
</rss>`,
'table',
'A'
);
expect(result.length).toStrictEqual(2);
expect((result as DataFrame).fields.length).toStrictEqual(5);
expect((result as DataFrame).fields[0].name).toStrictEqual('title');
expect((result as DataFrame).fields[0].values.toArray()[0]).toStrictEqual('Informational message: [RESOLVED] Internet connectivity');
expect((result as DataFrame).fields[2].values.toArray()[0]).toStrictEqual(new Date('Fri, 24 Dec 2021 10:41:17 PST'));
});
});
8 changes: 4 additions & 4 deletions src/app/UQLProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { uql } from 'uql';
import { isArray } from 'lodash';
import { DataFrame, FieldType, MutableDataFrame, toDataFrame } from '@grafana/data';
import { Datasource } from './../datasource';
import { normalizeURL } from './utils';
import { normalizeURL, toTimeSeriesMany } from './utils';
import { InfinityQueryFormat, InfinityUQLQuery } from '../types';

export class UQLProvider {
Expand All @@ -27,7 +27,7 @@ export class UQLProvider {
}
}

const sendAsDataFrame = (res: unknown, format: InfinityQueryFormat = 'table', refId: string): Promise<DataFrame> => {
const sendAsDataFrame = (res: unknown, format: InfinityQueryFormat = 'table', refId: string): Promise<DataFrame | DataFrame[]> => {
return new Promise((resolve, reject) => {
if (typeof res === 'number') {
let result = new MutableDataFrame({
Expand Down Expand Up @@ -57,7 +57,7 @@ const sendAsDataFrame = (res: unknown, format: InfinityQueryFormat = 'table', re
resolve(result);
} else if (typeof res === 'object' && isArray(res)) {
if (format === 'timeseries') {
resolve(toDataFrame(res)); // TODO: convert to multi frame results
resolve(toTimeSeriesMany([toDataFrame(res)]));
} else {
resolve(toDataFrame(res));
}
Expand All @@ -79,7 +79,7 @@ const sendAsDataFrame = (res: unknown, format: InfinityQueryFormat = 'table', re
});
};

export const applyUQL = (query: string, data: unknown, format: InfinityQueryFormat, refId: string): Promise<DataFrame> => {
export const applyUQL = (query: string, data: unknown, format: InfinityQueryFormat, refId: string): Promise<DataFrame | DataFrame[]> => {
if (!query) {
return sendAsDataFrame(data, format, refId);
}
Expand Down
25 changes: 0 additions & 25 deletions src/app/utils.test.ts

This file was deleted.

Loading

0 comments on commit 5b11aca

Please sign in to comment.