scripts/extract-timeseries-dashboard.js
// extract-timeseries-dashboard.js
//
// Token-efficient extractor: pulls every DQL query out of a Dynatrace dashboard
// and returns a compact TimeseriesDQLQuerySet[]. The caller (model or shell
// pipeline) never has to read the full dashboard JSON.
//
// Run with dtctl:
// dtctl exec function -f scripts/extract-timeseries-dashboard.js \
// --payload '{"id":"<dashboard-id-or-name>"}' -o json > queryset.json
//
// Payload shape:
// { "id": "<id-or-name>", "titleFilter": "...", "listOnly": true, "compact": true, "includeSkipped": true }
// - id required. Document id, or a name (resolved via Document API search).
// - titleFilter optional. Case-insensitive substring match against tile title;
// when set, only matching tiles are returned. Pass a string like
// "Hosts network traffic" or a /regex/ string ("/^cpu/i").
// - listOnly optional. When true, return tile names without DQL — lightweight
// disambiguation call. Response has "tiles" instead of "queries".
// - compact optional. When true, each query object contains only {id, title, dqlQuery}
// (drops description, visualization, isTimeseries). In listOnly mode,
// drops visualization, returning {id, title} only.
// - includeSkipped optional. When true, include the full skipped[] array in the response.
// Default: only skippedCount is returned.
//
// Response (TimeseriesDQLQuerySet[] wrapped in an envelope):
// {
// "ok": true,
// "documentId": "...", "documentName": "...", "documentVersion": 7,
// "queries": [
// { "id": "<tile-key>", "title": "...", "description": "...",
// "dqlQuery": "timeseries avg(dt.host.cpu.usage)",
// "visualization": "lineChart", "isTimeseries": true }
// ],
// "skipped": [ { "id": "...", "reason": "non-data tile" } ]
// }
//
// On error: { "ok": false, "error": { "code": "...", "message": "..." } }
import { documentsClient } from '@dynatrace-sdk/client-document';
// Match `timeseries` either at the start of the query or after a pipe. The
// regex is run after stripping leading `//` line comments / `/* … */` block
// comments / blank lines, so a query like
// `// some comment\ntimeseries sum(...)` still classifies as timeseries.
const TIMESERIES_RE = /(^|\|)\s*timeseries\b/i;
function stripLeadingCommentsAndWhitespace(query) {
let s = String(query);
while (true) {
const trimmed = s.replace(/^\s+/, '');
if (trimmed.startsWith('//')) {
const nl = trimmed.indexOf('\n');
s = nl === -1 ? '' : trimmed.slice(nl + 1);
continue;
}
if (trimmed.startsWith('/*')) {
const end = trimmed.indexOf('*/');
s = end === -1 ? '' : trimmed.slice(end + 2);
continue;
}
return trimmed;
}
}
function classify(query) {
if (typeof query !== 'string' || query.trim() === '') return false;
const stripped = stripLeadingCommentsAndWhitespace(query);
return TIMESERIES_RE.test(stripped);
}
// Accepts either a plain string (case-insensitive substring) or a /pattern/flags
// literal expressed as a string (e.g. "/^cpu/i") for regex matching.
//
// Tile titles often contain $variable placeholders (e.g. "CPU Usage $workload
// [Top $limit_graph_lines]"). When a direct substring match fails and the title
// has such placeholders, we convert the title into a regex by replacing each
// $word token with .* and test the caller's filter string against it. This lets
// a filter like "CPU Usage ppx-service [Top 500]" match the template title.
function compileTitleMatcher(filter) {
if (filter === undefined || filter === null || filter === '') return null;
const s = String(filter);
const m = s.match(/^\/(.+)\/([gimsuy]*)$/);
if (m) {
try {
// Strip stateful flags (g/y): re.test() is called per tile, and a stateful
// regex advances lastIndex between calls, causing intermittent false negatives.
const re = new RegExp(m[1], m[2].replace(/[gy]/g, ''));
return (title) => re.test(String(title ?? ''));
} catch (_) {
// fall through to substring match
}
}
const needle = s.toLowerCase();
return (title) => {
const raw = String(title ?? '');
// 1. Direct case-insensitive substring match (fast path).
if (raw.toLowerCase().includes(needle)) return true;
// 2. If the tile title has $variable placeholders, build a regex from it
// so resolved values like "ppx-service" or "500" match the template.
if (/\$\w/.test(raw)) {
const escaped = raw.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const pattern = escaped.replace(/\\\$\w+/g, '.*');
try {
if (new RegExp(pattern, 'i').test(s)) return true;
} catch (_) { /* ignore malformed pattern */ }
}
return false;
};
}
// The dashboard tile schema has moved around across versions. Try the known
// locations in order and return the first non-empty DQL string found.
function pickQuery(tile) {
const candidates = [
tile.query,
tile?.queryConfig?.query,
tile?.querySettings?.query,
tile?.data?.query,
tile?.input?.value,
];
for (const q of candidates) {
if (typeof q === 'string' && q.trim() !== '') return q;
}
return null;
}
function pickDescription(tile) {
return (
tile?.description ||
tile?.queryConfig?.description ||
tile?.visualizationSettings?.description ||
''
);
}
function pickVisualization(tile) {
return (
tile?.visualization ||
tile?.visualizationSettings?.visualizationType ||
tile?.type ||
''
);
}
// Tiles can be either an object (modern dashboards) keyed by string indices or
// an array (older variants). Normalise to [ {id, tile}, ... ].
function normaliseTiles(tilesRaw) {
if (!tilesRaw) return [];
if (Array.isArray(tilesRaw)) {
return tilesRaw.map((t, i) => ({ id: t?.id ?? String(i), tile: t }));
}
if (typeof tilesRaw === 'object') {
return Object.entries(tilesRaw).map(([k, v]) => ({ id: k, tile: v }));
}
return [];
}
// The SDK's getDocument returns only metadata + an empty content shell. The
// real dashboard body comes back from downloadDocumentContent as a `_Binary`
// wrapper whose `.data` is a ReadableStream of UTF-8 JSON bytes.
async function readBinaryStream(bin) {
const stream = bin?.data;
if (!stream || typeof stream.getReader !== 'function') {
throw new Error(
`downloadDocumentContent returned ${bin?.constructor?.name ?? typeof bin}, expected _Binary`,
);
}
const reader = stream.getReader();
const chunks = [];
let total = 0;
while (true) {
const { value, done } = await reader.read();
if (done) break;
chunks.push(value);
total += value.byteLength;
}
const buf = new Uint8Array(total);
let off = 0;
for (const c of chunks) {
buf.set(c, off);
off += c.byteLength;
}
return new TextDecoder().decode(buf);
}
async function fetchContent(id) {
const bin = await documentsClient.downloadDocumentContent({ id });
const text = await readBinaryStream(bin);
return JSON.parse(text);
}
async function fetchDocument(idOrName) {
let lookupError;
// Try as ID first — preset/template dashboards (e.g. `my.dynatrace.*`) only
// resolve this way.
try {
const raw = await documentsClient.getDocument({ id: idOrName });
const metadata = raw?.metadata ?? raw;
if (metadata?.id) {
const content = await fetchContent(metadata.id);
return { metadata, content };
}
lookupError = new Error(
`getDocument returned no id; raw response: ${JSON.stringify(raw).slice(0, 200)}`,
);
} catch (e) {
lookupError = e;
}
// Fall back to searching by exact name (only useful when the caller passed a
// name).
try {
const list = await documentsClient.listDocuments({
filter: `type=='dashboard' and name=='${idOrName.replace(/'/g, "\\'")}'`,
});
const items = list?.documents ?? list?.items ?? [];
if (items.length === 1) {
const metadata = items[0];
const content = await fetchContent(metadata.id);
return { metadata, content };
}
if (items.length > 1) {
const err = new Error(
`name is ambiguous (${items.length} matches): pass the document id instead`,
);
err.code = 'ambiguous';
throw err;
}
} catch (e) {
if (e.code === 'ambiguous') throw e;
}
const err = new Error(
`dashboard not found: ${idOrName} (lookup error: ${lookupError?.message ?? lookupError})`,
);
err.code = 'not_found';
throw err;
}
export default async function ({
id,
titleFilter,
listOnly = false,
compact = false,
includeSkipped = false,
} = {}) {
try {
if (!id) {
return {
ok: false,
error: { code: 'invalid_input', message: 'payload.id is required' },
};
}
const { metadata: doc, content } = await fetchDocument(id);
if (doc.type && doc.type !== 'dashboard') {
return {
ok: false,
error: {
code: 'wrong_type',
message: `document ${doc.id} is type "${doc.type}", expected "dashboard"`,
},
};
}
if (!content) {
return {
ok: false,
error: {
code: 'no_content',
message: `document ${doc.id} returned no content`,
},
};
}
const tiles = normaliseTiles(content?.tiles);
const titleMatches = compileTitleMatcher(titleFilter);
const queries = [];
const tileList = [];
const skipped = [];
for (const { id: tileId, tile } of tiles) {
if (!tile || typeof tile !== 'object') continue;
const tileType = tile.type ?? '';
if (['markdown', 'image', 'group'].includes(tileType)) {
skipped.push({ id: tileId, reason: `non-data tile (${tileType})` });
continue;
}
const tileTitle = tile.title ?? '';
if (titleMatches && !titleMatches(tileTitle)) {
skipped.push({ id: tileId, reason: 'title did not match filter' });
continue;
}
if (listOnly) {
tileList.push(compact
? { id: tileId, title: tileTitle }
: { id: tileId, title: tileTitle, visualization: pickVisualization(tile) });
continue;
}
const dqlQuery = pickQuery(tile);
if (!dqlQuery) {
skipped.push({ id: tileId, reason: 'no DQL query found' });
continue;
}
const isTimeseries = classify(dqlQuery);
if (!isTimeseries) {
skipped.push({ id: tileId, reason: 'not a timeseries query' });
continue;
}
queries.push(compact
? { id: tileId, title: tileTitle, dqlQuery }
: { id: tileId, title: tileTitle, description: pickDescription(tile), dqlQuery, visualization: pickVisualization(tile), isTimeseries });
}
const skippedSummary = includeSkipped
? { skipped }
: { skippedCount: skipped.length };
if (listOnly) {
return {
ok: true,
documentId: doc.id,
documentName: doc.name,
documentVersion: doc.version,
tiles: tileList,
...skippedSummary,
};
}
return {
ok: true,
documentId: doc.id,
documentName: doc.name,
documentVersion: doc.version,
queries,
...skippedSummary,
};
} catch (e) {
return {
ok: false,
error: {
code: e.code ?? 'internal_error',
message: e.message ?? String(e),
},
};
}
}
scripts/extract-timeseries-notebook.js
// extract-timeseries-notebook.js
//
// Same idea as extract-timeseries-dashboard.js but for notebooks. Notebooks
// store DQL inside sections/cells, not tiles, and the cell schema has a couple
// of variants — `sections[].content.input.value` (modern) and
// `cells[].state.input.value` (older). Both are handled.
//
// Run with dtctl:
// dtctl exec function -f scripts/extract-timeseries-notebook.js \
// --payload '{"id":"<notebook-id-or-name>"}' -o json > queryset.json
//
// Payload:
// { "id": "<id-or-name>", "titleFilter": "...", "compact": true, "includeSkipped": true }
// - titleFilter optional. Case-insensitive substring or "/regex/flags" literal.
// - compact optional. When true, each query object contains only {id, title, dqlQuery}.
// - includeSkipped optional. When true, return full skipped[] array. Default: skippedCount only.
//
// Response: same envelope as extract-timeseries-dashboard.js.
import { documentsClient } from '@dynatrace-sdk/client-document';
const TIMESERIES_RE = /(^|\|)\s*timeseries\b/i;
function stripLeadingCommentsAndWhitespace(query) {
let s = String(query);
while (true) {
const trimmed = s.replace(/^\s+/, '');
if (trimmed.startsWith('//')) {
const nl = trimmed.indexOf('\n');
s = nl === -1 ? '' : trimmed.slice(nl + 1);
continue;
}
if (trimmed.startsWith('/*')) {
const end = trimmed.indexOf('*/');
s = end === -1 ? '' : trimmed.slice(end + 2);
continue;
}
return trimmed;
}
}
function classify(query) {
if (typeof query !== 'string' || query.trim() === '') return false;
const stripped = stripLeadingCommentsAndWhitespace(query);
return TIMESERIES_RE.test(stripped);
}
function compileTitleMatcher(filter) {
if (filter === undefined || filter === null || filter === '') return null;
const s = String(filter);
const m = s.match(/^\/(.+)\/([gimsuy]*)$/);
if (m) {
try {
// Strip stateful flags (g/y): re.test() is called per cell, and a stateful
// regex advances lastIndex between calls, causing intermittent false negatives.
const re = new RegExp(m[1], m[2].replace(/[gy]/g, ''));
return (title) => re.test(String(title ?? ''));
} catch (_) {
// fall through to substring match
}
}
const needle = s.toLowerCase();
return (title) => String(title ?? '').toLowerCase().includes(needle);
}
function pickQuery(cell) {
const candidates = [
cell?.content?.input?.value,
cell?.state?.input?.value,
cell?.input?.value,
cell?.query,
cell?.content?.query,
];
for (const q of candidates) {
if (typeof q === 'string' && q.trim() !== '') return q;
}
return null;
}
function pickTitle(cell) {
return (
cell?.title ||
cell?.content?.title ||
cell?.state?.title ||
''
);
}
function pickDescription(cell) {
return (
cell?.description ||
cell?.content?.description ||
cell?.state?.description ||
''
);
}
function pickKind(cell) {
return (
cell?.content?.type ||
cell?.state?.type ||
cell?.type ||
''
);
}
function pickVisualization(cell) {
return (
cell?.content?.visualization ||
cell?.state?.visualization ||
cell?.visualization ||
''
);
}
// Notebooks have either `sections` (modern) or `cells` (older). Both come back
// as arrays. Normalise to [ {id, cell}, ... ].
function normaliseCells(content) {
const arr =
(Array.isArray(content?.sections) && content.sections) ||
(Array.isArray(content?.cells) && content.cells) ||
[];
return arr.map((c, i) => ({ id: c?.id ?? String(i), cell: c }));
}
// See extract-timeseries-dashboard.js — downloadDocumentContent returns a
// _Binary whose `.data` is a ReadableStream of UTF-8 JSON bytes.
async function readBinaryStream(bin) {
const stream = bin?.data;
if (!stream || typeof stream.getReader !== 'function') {
throw new Error(
`downloadDocumentContent returned ${bin?.constructor?.name ?? typeof bin}, expected _Binary`,
);
}
const reader = stream.getReader();
const chunks = [];
let total = 0;
while (true) {
const { value, done } = await reader.read();
if (done) break;
chunks.push(value);
total += value.byteLength;
}
const buf = new Uint8Array(total);
let off = 0;
for (const c of chunks) {
buf.set(c, off);
off += c.byteLength;
}
return new TextDecoder().decode(buf);
}
async function fetchContent(id) {
const bin = await documentsClient.downloadDocumentContent({ id });
const text = await readBinaryStream(bin);
return JSON.parse(text);
}
async function fetchDocument(idOrName) {
let lookupError;
try {
const raw = await documentsClient.getDocument({ id: idOrName });
const metadata = raw?.metadata ?? raw;
if (metadata?.id) {
const content = await fetchContent(metadata.id);
return { metadata, content };
}
lookupError = new Error(
`getDocument returned no id; raw response: ${JSON.stringify(raw).slice(0, 200)}`,
);
} catch (e) {
lookupError = e;
}
try {
const list = await documentsClient.listDocuments({
filter: `type=='notebook' and name=='${idOrName.replace(/'/g, "\\'")}'`,
});
const items = list?.documents ?? list?.items ?? [];
if (items.length === 1) {
const metadata = items[0];
const content = await fetchContent(metadata.id);
return { metadata, content };
}
if (items.length > 1) {
const err = new Error(
`name is ambiguous (${items.length} matches): pass the document id instead`,
);
err.code = 'ambiguous';
throw err;
}
} catch (e) {
if (e.code === 'ambiguous') throw e;
}
const err = new Error(
`notebook not found: ${idOrName} (lookup error: ${lookupError?.message ?? lookupError})`,
);
err.code = 'not_found';
throw err;
}
export default async function ({
id,
titleFilter,
compact = false,
includeSkipped = false,
} = {}) {
try {
if (!id) {
return {
ok: false,
error: { code: 'invalid_input', message: 'payload.id is required' },
};
}
const { metadata: doc, content } = await fetchDocument(id);
if (doc.type && doc.type !== 'notebook') {
return {
ok: false,
error: {
code: 'wrong_type',
message: `document ${doc.id} is type "${doc.type}", expected "notebook"`,
},
};
}
if (!content) {
return {
ok: false,
error: {
code: 'no_content',
message: `document ${doc.id} returned no content`,
},
};
}
const cells = normaliseCells(content);
const titleMatches = compileTitleMatcher(titleFilter);
const queries = [];
const skipped = [];
for (const { id: cellId, cell } of cells) {
if (!cell || typeof cell !== 'object') continue;
const kind = pickKind(cell);
// Notebooks support many cell kinds: dql, markdown, sql, businessEvent
// queries, etc. Anything not a DQL/query cell is skipped.
if (kind && !['dql', 'query', 'data'].includes(kind)) {
skipped.push({ id: cellId, reason: `non-DQL cell (${kind})` });
continue;
}
const cellTitle = pickTitle(cell);
if (titleMatches && !titleMatches(cellTitle)) {
skipped.push({ id: cellId, reason: 'title did not match filter' });
continue;
}
const dqlQuery = pickQuery(cell);
if (!dqlQuery) {
skipped.push({ id: cellId, reason: 'no DQL query found' });
continue;
}
const isTimeseries = classify(dqlQuery);
if (!isTimeseries) {
skipped.push({ id: cellId, reason: 'not a timeseries query' });
continue;
}
queries.push(compact
? { id: cellId, title: cellTitle, dqlQuery }
: { id: cellId, title: cellTitle, description: pickDescription(cell), dqlQuery, visualization: pickVisualization(cell), isTimeseries });
}
const skippedSummary = includeSkipped
? { skipped }
: { skippedCount: skipped.length };
return {
ok: true,
documentId: doc.id,
documentName: doc.name,
documentVersion: doc.version,
queries,
...skippedSummary,
};
} catch (e) {
return {
ok: false,
error: {
code: e.code ?? 'internal_error',
message: e.message ?? String(e),
},
};
}
}
scripts/run-analyzer.js
// run-analyzer.js
//
// Generic Davis analyzer runner for a TimeseriesDQLQuerySet
// (the output of extract-timeseries-dashboard.js / extract-timeseries-notebook.js).
//
// Wraps the common execute → poll → concurrent pattern so each Davis analyzer
// call doesn't need its own script.
//
// Run with dtctl (no jq needed — the script normalizes the array / envelope /
// dtctl-output shapes internally, and unwraps the -o json {"result":{...}} wrapper):
// dtctl exec function -f scripts/extract-timeseries-dashboard.js \
// --payload '{"id":"<dashboard-id>"}' -o json > queryset.json
//
// dtctl exec function -f scripts/run-analyzer.js \
// --payload '{
// "analyzerName": "dt.statistics.NoveltyScoreAnalyzer",
// "queries": '"$(cat queryset.json)"'
// }' -o json
//
// For large querysets, build a payload file and use --data instead of inlining:
// { printf '{"analyzerName":"dt.statistics.NoveltyScoreAnalyzer","queries":'; \
// cat queryset.json; printf '}'; } > payload.json
// dtctl exec function -f scripts/run-analyzer.js --data payload.json -o json
//
// Payload:
// {
// "analyzerName": "dt.statistics.NoveltyScoreAnalyzer" (required)
// "queries": TimeseriesDQLQuery[] (required)
// Accepts the raw array OR the extractor envelope (reads .queries).
// "timeframe": { "startTime": "now-1h", "endTime": "now" }
// also accepts { "from": "...", "to": "..." }
// (optional, default last 1 hour)
// "analyzerParams": { ... }
// Extra fields merged into each analyzer call body.
// Use for analyzer-specific knobs that don't belong in
// generalParameters. Examples:
// Anomaly detection: { "trainingTimeframe": { "startTime": "now-8d", "endTime": "now-1d" } }
// Novelty scoring: { "detectionMode": "ALL", "minNoveltyScore": 0 }
// (optional, default {})
// "metricQuery": "timeseries avg(dt.host.cpu.usage)"
// When set, switches to correlation mode.
// For Pearson analyzers (name contains "pearson"):
// referenceTimeseries = metricQuery (the primary)
// toCorrelateTimeseries = each query in the set
// For other correlation analyzers:
// timeSeriesData = metricQuery (the primary)
// correlatedTimeSeriesData = each query in the set
// When unset, each query is used as timeSeriesData independently.
// (optional)
// "metricQueryFrom": <full output of a previous run-analyzer.js call>
// Alternative to metricQuery: accepts the raw dtctl output
// (or unwrapped result) of a prior run and picks the
// highest-scored result's dqlQuery as the primary.
// Useful for chaining anomaly detection → correlation without
// a local jq step. metricQuery takes precedence if both are set.
// (optional)
// "variables": { "varName": "value", ... }
// Substitutes $varName tokens in DQL before execution.
// Build from URL vfilter_* params: strip the "vfilter_" prefix.
// Trailing * wildcards on values are stripped automatically.
// (optional)
// "maxConcurrent": 4 (optional, default 4)
// "minScore": 0.5
// When set, only results whose detected score >= minScore are returned.
// Results with no detectable score are dropped. (optional)
// "scoreField": "noveltyScore"
// Explicit field name to read the score from. When omitted, auto-detects
// from: noveltyScore, anomalyScore, correlationCoefficient, correlation,
// coefficient, or any field whose name contains "score". (optional)
// }
//
// Response:
// {
// "ok": true,
// "checkedAt": "...",
// "analyzerName": "...",
// "summary": { "checked": N, "completed": K, "errors": E },
// "results": [
// {
// "id": "...", "title": "...", "dqlQuery": "...",
// "output": <raw analyzer output>,
// "executionStatus": "COMPLETED"
// }
// ],
// "errors": [ { "id": "...", "error": "..." } ]
// }
import { analyzersClient } from '@dynatrace-sdk/client-davis-analyzers';
const DEFAULT_TIMEFRAME = { startTime: 'now-1h', endTime: 'now' };
// ─── Analyzer execute + poll ──────────────────────────────────────────────────
async function pollAnalyzer(analyzerName, requestToken, maxAttempts = 30) {
for (let i = 0; i < maxAttempts; i++) {
const poll = await analyzersClient.pollAnalyzerExecution({ analyzerName, requestToken });
if (poll?.result?.executionStatus === 'COMPLETED') return poll;
if (poll?.result?.executionStatus === 'FAILED') {
const msg = poll?.result?.errorMessage ?? 'analyzer execution failed';
const err = new Error(msg);
err.code = 'analyzer_failed';
throw err;
}
await new Promise((r) => setTimeout(r, 2000));
}
const err = new Error('analyzer execution timed out');
err.code = 'timeout';
throw err;
}
async function executeAnalyzer(analyzerName, body) {
const res = await analyzersClient.executeAnalyzer({ analyzerName, body });
if (res?.requestToken && res?.result === undefined) {
return await pollAnalyzer(analyzerName, res.requestToken);
}
// Immediate (non-polled) response: surface a terminal FAILED status as an error
// rather than returning it as a successful item. (The polled path already throws
// on FAILED via pollAnalyzer; RUNNING/COMPLETED are left to the caller.)
if (res?.result?.executionStatus === 'FAILED') {
const err = new Error(res?.result?.errorMessage ?? 'analyzer execution failed');
err.code = 'analyzer_failed';
throw err;
}
return res;
}
// ─── Helpers ──────────────────────────────────────────────────────────────────
function normalizeTimeframe(tf) {
if (!tf) return null;
// Fall back to defaults per-field so a partial timeframe (e.g. only startTime,
// or an empty {}) never yields undefined fields in the analyzer request body.
return {
startTime: tf.startTime ?? tf.from ?? DEFAULT_TIMEFRAME.startTime,
endTime: tf.endTime ?? tf.to ?? DEFAULT_TIMEFRAME.endTime,
};
}
function normalizeQueries(payload) {
if (Array.isArray(payload)) return payload;
if (Array.isArray(payload?.queries)) return payload.queries;
if (Array.isArray(payload?.result?.queries)) return payload.result.queries; // dtctl -o json wrapper
return null;
}
// Accepts the full output of a previous run-analyzer.js call (raw dtctl output or
// unwrapped result) and returns the dqlQuery of the highest-scored result.
function topDqlFromResults(input, scoreField) {
const results = input?.result?.results ?? input?.results;
if (!Array.isArray(results) || results.length === 0) return null;
const sorted = [...results].sort((a, b) => {
const sa = extractScore(a.output, scoreField) ?? -Infinity;
const sb = extractScore(b.output, scoreField) ?? -Infinity;
return sb - sa;
});
return sorted[0]?.dqlQuery ?? null;
}
// Strip `| timeframe ...` pipeline stages embedded in dashboard DQL so that
// caller-supplied defaultTimeframe parameters take effect for historical windows.
// Only strips when the caller provides an absolute (non-now) start timestamp.
function stripEmbeddedTimeframe(dql, startTime) {
if (!startTime || typeof startTime !== 'string' || startTime.startsWith('now')) return dql;
return dql.replace(/\|\s*timeframe\b[^|]*/gi, ' ').replace(/\s+/g, ' ').trim();
}
// Substitute $varName tokens from URL vfilter_* parameters before execution.
// Values that look like plain numbers are inserted unquoted; everything else
// is JSON-quoted. Trailing wildcard (*) is stripped (DQL in() doesn't support
// wildcards). Unresolved $variables are cleaned up from filter clauses to avoid
// silently dropping entity filters or generating DQL errors.
function applyVariables(dql, variables) {
if (!variables || typeof variables !== 'object') return dql;
let result = dql;
for (const [name, rawVal] of Object.entries(variables)) {
const val = String(rawVal).replace(/\*$/, '');
const quoted = /^\d+(\.\d+)?$/.test(val) ? val : JSON.stringify(val);
const re = new RegExp(
'\\$' + name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '(?=[^a-zA-Z0-9_]|$)',
'g',
);
result = result.replace(re, quoted);
}
// Drop filter clauses referencing still-unresolved $variables.
const EXPR = '[^$]*?';
const VARNAME = '\\$[a-zA-Z_][a-zA-Z0-9_]*';
result = result.replace(new RegExp(`\\s*\\band\\b\\s+(?:not\\s+)?in\\s*\\(${EXPR},\\s*${VARNAME}\\s*\\)`, 'gi'), '');
result = result.replace(new RegExp(`(?:not\\s+)?in\\s*\\(${EXPR},\\s*${VARNAME}\\s*\\)\\s*\\band\\b\\s*`, 'gi'), '');
result = result.replace(new RegExp(`\\bnot\\s+in\\s*\\(${EXPR},\\s*${VARNAME}\\s*\\)`, 'gi'), 'false');
result = result.replace(new RegExp(`\\bin\\s*\\(${EXPR},\\s*${VARNAME}\\s*\\)`, 'gi'), 'true');
result = result.replace(/\$[a-zA-Z_][a-zA-Z0-9_]*/g, 'null');
return result;
}
const SCORE_FIELDS = ['noveltyScore', 'anomalyScore', 'correlationCoefficient', 'correlation', 'coefficient'];
// Analyzer outputs are nested (arrays of per-dimension objects, and scores can sit
// one or two levels in — e.g. correlationCoefficient in each array element, or
// novelties[].noveltyScore). Walk the structure looking for an explicit scoreField
// or one of the known score fields, then fall back to any top-level "*score*" number.
function findKnownScore(node, scoreField) {
if (node == null || typeof node !== 'object') return null;
if (Array.isArray(node)) {
for (const el of node) {
const s = findKnownScore(el, scoreField);
if (s != null) return s;
}
return null;
}
if (scoreField && typeof node[scoreField] === 'number') return node[scoreField];
for (const field of SCORE_FIELDS) {
if (typeof node[field] === 'number') return node[field];
}
for (const v of Object.values(node)) {
if (v && typeof v === 'object') {
const s = findKnownScore(v, scoreField);
if (s != null) return s;
}
}
return null;
}
function extractScore(output, scoreField) {
const known = findKnownScore(output, scoreField);
if (known != null) return known;
// Best-effort fallback for unknown analyzers: any top-level "*score*" number.
if (output && typeof output === 'object' && !Array.isArray(output)) {
for (const [k, v] of Object.entries(output)) {
if (k.toLowerCase().includes('score') && typeof v === 'number') return v;
}
}
return null;
}
async function mapConcurrent(items, n, fn) {
const out = new Array(items.length);
let cursor = 0;
const workers = Array.from({ length: Math.min(n, items.length) }, async () => {
while (true) {
const i = cursor++;
if (i >= items.length) return;
out[i] = await fn(items[i], i);
}
});
await Promise.all(workers);
return out;
}
// ─── Main export ──────────────────────────────────────────────────────────────
export default async function ({
analyzerName,
queries: queriesInput,
timeframe,
analyzerParams = {},
metricQuery,
metricQueryFrom,
variables,
maxConcurrent = 4,
minScore,
scoreField,
} = {}) {
try {
if (typeof analyzerName !== 'string' || analyzerName.trim() === '') {
return {
ok: false,
error: { code: 'invalid_input', message: 'payload.analyzerName must be a non-empty string' },
};
}
// Resolve metricQuery from a previous run's output when not given directly.
if (metricQuery == null && metricQueryFrom != null) {
metricQuery = topDqlFromResults(metricQueryFrom, scoreField);
if (metricQuery == null) {
return {
ok: false,
error: { code: 'invalid_input', message: 'metricQueryFrom contained no results with a dqlQuery' },
};
}
}
const queries = normalizeQueries(queriesInput);
if (!Array.isArray(queries) || queries.length === 0) {
return {
ok: false,
error: {
code: 'invalid_input',
message: 'payload.queries must be a non-empty TimeseriesDQLQuerySet (array or extractor envelope)',
},
};
}
const tf = normalizeTimeframe(timeframe) ?? DEFAULT_TIMEFRAME;
const work = queries.filter((q) => q && typeof q.dqlQuery === 'string' && q.isTimeseries !== false);
// The primary/correlation query gets the same treatment as the per-tile queries:
// strip embedded `| timeframe` for absolute windows and substitute $variables.
// Otherwise a primary sourced from an extracted tile (or metricQueryFrom) could
// still carry $var tokens or a different embedded window than the correlated set.
const resolvedMetricQuery = metricQuery != null
? applyVariables(stripEmbeddedTimeframe(metricQuery, tf.startTime), variables)
: null;
// Sanitize fan-out: coerce to a positive integer so a bad value (0, negative,
// non-numeric) can't silently produce zero workers and an empty result set.
const concurrency = Math.max(1, Math.floor(Number(maxConcurrent)) || 4);
const results = await mapConcurrent(work, concurrency, async (q) => {
try {
const resolvedDQL = applyVariables(
stripEmbeddedTimeframe(q.dqlQuery, tf.startTime),
variables,
);
const pearsonMode = /pearson/i.test(analyzerName);
const body = resolvedMetricQuery
? pearsonMode
? {
referenceTimeseries: resolvedMetricQuery,
toCorrelateTimeseries: resolvedDQL,
generalParameters: { timeframe: tf, resolveDimensionalQueryData: true },
...analyzerParams,
}
: {
timeSeriesData: resolvedMetricQuery,
correlatedTimeSeriesData: resolvedDQL,
generalParameters: { timeframe: tf, resolveDimensionalQueryData: true },
...analyzerParams,
}
: {
timeSeriesData: resolvedDQL,
generalParameters: { timeframe: tf, resolveDimensionalQueryData: true },
...analyzerParams,
};
const raw = await executeAnalyzer(analyzerName, body);
return {
ok: true,
item: {
id: q.id,
title: q.title ?? '',
dqlQuery: q.dqlQuery,
output: raw?.result?.output ?? null,
executionStatus: raw?.result?.executionStatus ?? 'COMPLETED',
},
};
} catch (e) {
return { ok: false, id: q.id, error: e.message ?? String(e) };
}
});
const completed = [];
const errors = [];
for (const r of results) {
if (r.ok) completed.push(r.item);
else errors.push({ id: r.id, error: r.error });
}
const filtered = minScore != null
? completed.filter((item) => {
const score = extractScore(item.output, scoreField);
return score != null && score >= minScore;
})
: completed;
return {
ok: true,
checkedAt: new Date().toISOString(),
analyzerName,
summary: { checked: work.length, completed: completed.length, errors: errors.length },
results: filtered,
errors,
};
} catch (e) {
return {
ok: false,
error: { code: e.code ?? 'internal_error', message: e.message ?? String(e) },
};
}
}
SKILL.md
---
name: dt-obs-analytics
description: >-
Analyze dashboards and notebooks using Davis analyzers — anomaly detection, novelty scoring,
and correlation. Use when the user references a specific Dynatrace dashboard or notebook
(by URL, UUID, or name) and asks what it shows, which DQL queries it runs, whether a tile
looks off, or wants to find anomalies, score novelty, or correlate its metrics.
The trigger is a dashboard or notebook as the data source, not a general DQL question.
This skill extracts timeseries queries efficiently without reading the full raw document JSON,
then optionally runs Davis analyzers on the extracted metrics.
Trigger phrases: "what's wrong on this dashboard", "analyze this notebook", "find anomalies",
"novelty score", "correlate metrics", "extract DQL from dashboard", "dashboard URL", "tile",
"run-analyzer", "timeseries extraction", "Davis analyzer".
license: Apache-2.0
---
# Analytics — Dashboard & Notebook Query Extraction
A pipeline of three platform JavaScript scripts under `scripts/` — two extractors feeding a shared analyzer runner:
```
scripts/extract-timeseries-dashboard.js ──┐
scripts/extract-timeseries-notebook.js ──┴──► queryset.json ──► scripts/run-analyzer.js (any Davis analyzer)
```
Each script is invoked via:
```bash
dtctl exec function -f scripts/<script>.js --payload '<json>' -o json
```
`-o json` wraps the function return value under `.result`. When you need to inspect the result, read it directly from the output — no `jq` required. Pass the full raw output as `queries` to `run-analyzer.js` and it unwraps automatically.
## Parsing a dashboard URL
When the entry point is a Dynatrace dashboard URL, extract the three components the scripts need:
```
https://<tenant>/ui/apps/dynatrace.dashboards/dashboard/<ID>#from=<from>&to=<to>&vfilter_<name>=<val>...
```
| URL part | Script destination |
|---|---|
| Path segment after `/dashboard/` (before `#`) | `id` in extract-timeseries-dashboard.js payload |
| `#from=` value (URL-decode `%3A` → `:`) | `timeframe.startTime` in run-analyzer.js (only needed if running analysis) |
| `#to=` value (URL-decode) | `timeframe.endTime` in run-analyzer.js (only needed if running analysis) |
| `vfilter_<name>=<value>` params | `variables` map in run-analyzer.js (strip `vfilter_` prefix) |
The timeframe in the URL fragment is the dashboard's *display* window. It is **not** injected into the extracted DQL — the **extractor** returns the DQL verbatim with its original `$variable` tokens and any embedded `| timeframe` clauses intact. Use the parsed `from`/`to` values only when calling `run-analyzer.js` to set the analysis window. If the user just wants to list the queries, the timeframe is informational only.
Note: when you pass `run-analyzer.js` an **absolute** `timeframe.startTime` (not a `now...` expression), it strips any embedded `| timeframe ...` stage from the DQL before analysis, so the analyzer honors your requested window rather than the query's baked-in one. For relative (`now...`) windows the embedded `| timeframe` is left intact. This means the query actually analyzed can differ from the extracted text — expected behavior, noted here so results line up with the window you asked for.
Quick bash parse (pure bash + sed/awk — no python needed):
```bash
DASHBOARD_URL="https://abc123.apps.dynatrace.com/ui/apps/dynatrace.dashboards/dashboard/5bea16c7-029b-43b6-9735-459db2d25bbf#from=2026-05-28T04%3A00Z&to=2026-05-28T05%3A00Z&vfilter_host_group=prod&vfilter_workload=my-svc"
# Minimal URL-decoder: turn %XX into \xXX and let printf interpret it.
urldecode() { local s="${1//+/ }"; printf '%b' "${s//%/\\x}"; }
DOC_ID=$(echo "$DASHBOARD_URL" | sed 's/#.*//' | awk -F/ '{print $NF}')
FROM=$(urldecode "$(echo "$DASHBOARD_URL" | sed -n 's/.*[#&]from=\([^&]*\).*/\1/p')")
TO=$(urldecode "$(echo "$DASHBOARD_URL" | sed -n 's/.*[#&]to=\([^&]*\).*/\1/p')")
HOST_GROUP=$(echo "$DASHBOARD_URL" | sed -n 's/.*[#&]vfilter_host_group=\([^&]*\).*/\1/p')
WORKLOAD=$(echo "$DASHBOARD_URL" | sed -n 's/.*[#&]vfilter_workload=\([^&]*\).*/\1/p')
```
For notebooks: path segment after `/notebook/`, or `#share=` value for `/document/v0/#share=<ID>` links.
## Step 1 — Extract queries
### From a dashboard
```bash
# All tiles
dtctl exec function -f scripts/extract-timeseries-dashboard.js \
--payload '{"id":"<dashboard-id-or-name>"}' -o json
# Only tiles whose title matches a name the user mentioned (e.g. "CPU usage", "Kafka lag")
dtctl exec function -f scripts/extract-timeseries-dashboard.js \
--payload '{"id":"<dashboard-id>","titleFilter":"CPU usage"}' -o json
```
**When the user names a specific tile, chart, or section**, pass its name as `titleFilter` rather than extracting the full dashboard. `titleFilter` is a case-insensitive substring or `/regex/flags` pattern. This keeps the queryset small and focused.
**When it is not clear which tile(s) the user wants**, do NOT extract all DQL — dashboards can have 20–50 tiles and returning all queries causes significant context bloat. Instead use a two-step flow:
1. List tile names with `listOnly: true` (no DQL, just titles):
```bash
dtctl exec function -f scripts/extract-timeseries-dashboard.js \
--payload '{"id":"<dashboard-id>","listOnly":true}' -o json
# Returns: {"result":{"ok":true,"tiles":[{"id":"...","title":"CPU Usage","visualization":"lineChart"},...]}}
```
2. Show the tile names to the user and ask which tile(s) they mean.
3. Re-run with `titleFilter` for only the tile(s) of interest.
This avoids pulling 20–50 DQL queries into context when only 1–2 are relevant.
Payload knobs:
- `id` (required) — dashboard ID (UUID) or exact name. Preset IDs like `my.dynatrace.infraops.preview.*` work.
- `titleFilter` — case-insensitive substring (`"CPU usage"`) or `/regex/flags` (`"/^kafka/i"`).
- `listOnly` — when `true`, returns `tiles: [{id, title, visualization}]` without DQL. Use for disambiguation.
- `compact` — when `true`, returns only `{id, title, dqlQuery}` per tile (drops description, visualization, isTimeseries). Saves ~40% per-tile tokens. In `listOnly` mode, drops visualization too.
- `includeSkipped` — when `true`, returns the full `skipped[]` array. Default: only `skippedCount` is returned.
Response envelope:
```json
{
"ok": true,
"documentId": "...", "documentName": "...", "documentVersion": 7,
"queries": [
{ "id": "<tile-key>", "title": "...", "description": "...",
"dqlQuery": "timeseries avg(dt.host.cpu.usage)",
"visualization": "lineChart", "isTimeseries": true }
],
"skipped": [{ "id": "...", "reason": "non-data tile (markdown)" }]
}
```
On failure: `{ "ok": false, "error": { "code": "...", "message": "..." } }`.
### From a notebook
Same envelope, different schema walk:
```bash
# All cells
dtctl exec function -f scripts/extract-timeseries-notebook.js \
--payload '{"id":"<notebook-id-or-name>"}' -o json
# A specific section (if cell titles are set)
dtctl exec function -f scripts/extract-timeseries-notebook.js \
--payload '{"id":"<notebook-id>","titleFilter":"JVM memory"}' -o json
```
Notebook cells often have empty titles — prefer addressing cells by `id` from the envelope if targeting a specific one.
## Step 2 — Run an analyzer
Save the extractor output to a file, then pass it via shell substitution — the shell reads the file, so the JSON never enters the model's context:
```bash
# Run extractor, save output
dtctl exec function -f scripts/extract-timeseries-dashboard.js \
--payload '{"id":"<id>","titleFilter":"CPU usage","compact":true}' -o json > queryset.json
# Shell substitution: $(cat queryset.json) is expanded by the shell, not the model
dtctl exec function -f scripts/run-analyzer.js \
--payload '{
"analyzerName": "dt.statistics.NoveltyScoreAnalyzer",
"queries": '"$(cat queryset.json)"',
"timeframe": { "startTime": "...", "endTime": "..." },
"analyzerParams": { ... }
}' -o json
```
`run-analyzer.js` unwraps the `{"result":{...}}` dtctl envelope automatically — pass the raw saved output as-is. No `jq` or parsing step is needed: normalization of the array / envelope / dtctl-output shapes happens inside the script.
For large querysets (many tiles), the inline `$(cat ...)` form can hit shell argument-length limits. Build the payload file and use dtctl's `--data` flag instead — still no `jq` and still out of model context:
```bash
{ printf '{"analyzerName":"dt.statistics.NoveltyScoreAnalyzer","timeframe":{"startTime":"now-1h","endTime":"now"},"queries":'
cat queryset.json
printf '}'; } > payload.json
dtctl exec function -f scripts/run-analyzer.js --data payload.json -o json
```
Key payload knobs for `run-analyzer.js`:
- `minScore` — drop results below this threshold (e.g. `0.5`). Auto-detects score field from `noveltyScore`, `anomalyScore`, `correlationCoefficient`, `correlation`, `coefficient`. Pass `scoreField` to override.
- `scoreField` — explicit field name to read score from (e.g. `"noveltyScore"`).
The `queries` field accepts any of: a raw array, the extractor envelope (`{queries:[...]}`), or the full dtctl output (`{"result":{"queries":[...]}}`). All three are normalized automatically.
### Common analyzers
| Goal | analyzerName | analyzerParams |
|---|---|---|
| Find anomalous metrics | `dt.statistics.anomaly_detection.SeasonalBaselineAnomalyDetectionAnalyzer` | `{ "trainingTimeframe": { "startTime": "now-8d", "endTime": "now-1d" } }` (optional) |
| Score how novel each metric is | `dt.statistics.NoveltyScoreAnalyzer` | `{ "detectionMode": "ALL", "minNoveltyScore": 0 }` (optional) |
| Correlate against a primary metric | `dt.statistics.SimplePearsonCorrelationAnalyzer` | set `metricQuery` instead (changes call shape) |
### Correlation mode
Pass `metricQuery` to correlate every query in the set against a single primary DQL string:
```bash
dtctl exec function -f scripts/run-analyzer.js \
--payload '{
"analyzerName": "dt.statistics.SimplePearsonCorrelationAnalyzer",
"queries": '"$(cat queryset.json)"',
"metricQuery": "<dqlQuery of the primary tile, copied from extractor output>",
"timeframe": { "startTime": "...", "endTime": "..." }
}' -o json
```
When chaining from a previous analyzer run (e.g. anomaly detection → correlation), use `metricQueryFrom` instead. The script picks the highest-scored result's `dqlQuery` automatically:
```bash
dtctl exec function -f scripts/run-analyzer.js \
--payload '{
"analyzerName": "dt.statistics.SimplePearsonCorrelationAnalyzer",
"queries": '"$(cat queryset.json)"',
"metricQueryFrom": '"$(cat findings.json)"',
"timeframe": { "startTime": "...", "endTime": "..." }
}' -o json
```
`metricQuery` takes precedence if both are set.
### Variable substitution
Dashboard queries often contain `$variable` tokens (from URL `vfilter_*` params). Pass them via `variables` to substitute before execution:
```bash
dtctl exec function -f scripts/run-analyzer.js \
--payload '{
"analyzerName": "dt.statistics.anomaly_detection.SeasonalBaselineAnomalyDetectionAnalyzer",
"queries": [...],
"timeframe": { "startTime": "2026-05-28T04:00Z", "endTime": "2026-05-28T05:00Z" },
"variables": { "host_group": "prod", "workload": "my-svc" }
}' -o json
```
Build `variables` from `vfilter_*` URL params by stripping the `vfilter_` prefix. Trailing `*` wildcards are stripped automatically. Unresolved tokens are cleaned up from DQL filter clauses rather than left to error.
### Response shape
```json
{
"ok": true,
"checkedAt": "...",
"analyzerName": "...",
"summary": { "checked": 12, "completed": 11, "errors": 1 },
"results": [
{
"id": "tile-key", "title": "CPU usage", "dqlQuery": "...",
"output": <raw analyzer output>,
"executionStatus": "COMPLETED"
}
],
"errors": [ { "id": "...", "error": "..." } ]
}
```
The `output` field is the raw analyzer result. Interpret it based on the analyzer:
- **Anomaly detection**: look for `anomalyScore`, `anomalies[]`, or `raisedAlerts[]` in each output entry. Score ≥ 0.7 → abnormal, ≥ 0.4 → borderline.
- **Novelty**: look for `noveltyScore` (or the closest score-like numeric field). Score ≥ 0.7 → novel.
- **Correlation**: look for `correlationCoefficient`. Sort by `|correlationCoefficient|` descending; drop entries where `|correlationCoefficient| < 0.5`.
## End-to-end: "what's abnormal on this dashboard?"
```bash
# 1. Extract queries — shell reads file, JSON stays out of model context
dtctl exec function -f scripts/extract-timeseries-dashboard.js \
--payload '{"id":"5bea16c7-029b-43b6-9735-459db2d25bbf","compact":true}' \
-o json > queryset.json
# 2. Run anomaly detection — $(cat queryset.json) expanded by shell, not model
dtctl exec function -f scripts/run-analyzer.js \
--payload '{
"analyzerName": "dt.statistics.anomaly_detection.SeasonalBaselineAnomalyDetectionAnalyzer",
"queries": '"$(cat queryset.json)"',
"timeframe": { "startTime": "2026-05-28T04:00Z", "endTime": "2026-05-28T05:00Z" },
"variables": { "host_group": "prod", "workload": "my-svc" }
}' -o json > findings.json
# 3. Correlate — metricQueryFrom picks the top finding automatically
dtctl exec function -f scripts/run-analyzer.js \
--payload '{
"analyzerName": "dt.statistics.SimplePearsonCorrelationAnalyzer",
"queries": '"$(cat queryset.json)"',
"metricQueryFrom": '"$(cat findings.json)"',
"timeframe": { "startTime": "2026-05-28T04:00Z", "endTime": "2026-05-28T05:00Z" }
}' -o json
```
## Verifying a single extracted query
Read the `dqlQuery` field from the extractor output and pass it directly:
```bash
dtctl query --query "<dqlQuery copied from extractor output>" -o json | head -40
```
## Gotchas
- **Never read raw dashboard JSON yourself.** `dtctl get dashboard <id> -o json` is typically 50–200 KB. The extractor reads it on the platform and returns a compact envelope (~5–15 KB).
- **Never extract all tiles when only one is needed.** A 50-tile dashboard returns 50 DQL queries into context. If the user names a tile, use `titleFilter`. If it's ambiguous, use `listOnly: true` first to ask which tile — then extract only that one.
- **Variables are required for filtered dashboards.** Queries with unsubstituted `$variable` tokens silently drop entity filters (e.g. `in(field, $undefined)` evaluates to `true`). Always pass `variables` when the URL has `vfilter_*` params.
- **Schema drift.** If a tile lands in `skipped` with reason `no DQL query found`, the dashboard schema has a query location the extractor doesn't know about — add it to the `pickQuery` candidate list in the script.
- **Analyzer availability.** Not all Davis analyzers are available on every tenant. If a call comes back with `Could not find an analyzer with name '...'` or `is not a function`, list what's actually registered: `dtctl get analyzers -o json`.
- **Statistical fallback removed.** `run-analyzer.js` only calls Davis analyzers. For historical anomaly detection, pass a long `trainingTimeframe` via `analyzerParams` (e.g. `{ "trainingTimeframe": { "startTime": "now-30d", "endTime": "now-1d" } }`), or query the DQL directly.
- **Comments in queries.** Queries starting with `// comment` lines are classified correctly by the extractor (leading line/block comments are stripped before the `timeseries` check).
## Scripts reference
- [scripts/extract-timeseries-dashboard.js](scripts/extract-timeseries-dashboard.js) — extracts timeseries DQL from a dashboard
- [scripts/extract-timeseries-notebook.js](scripts/extract-timeseries-notebook.js) — same for notebooks
- [scripts/run-analyzer.js](scripts/run-analyzer.js) — generic Davis analyzer runner