{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 33, "links": [ { "icon": "external link", "includeVars": true, "keepTime": true, "tags": [], "title": "Model Monitoring - Performance", "type": "link", "url": "/d/9CazA-UGz/model-monitoring-performance" }, { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [], "title": "Model Monitoring - Overview", "type": "link", "url": "d/g0M4uh0Mz/model-monitoring-overview" } ], "liveNow": false, "panels": [ { "datasource": "iguazio", "description": "", "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "auto", "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "First Request" }, "properties": [ { "id": "unit", "value": "dateTimeFromNow" }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "Last Request" }, "properties": [ { "id": "unit", "value": "dateTimeFromNow" }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "Endpoint ID" }, "properties": [ { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "Model" }, "properties": [ { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "Function URI" }, "properties": [ { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "Model Class" }, "properties": [ { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "Predictions/s (5 minute avg)" }, "properties": [ { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "Average Latency (1 hour)" }, "properties": [ { "id": "custom.align", "value": "center" }, { "id": "unit", "value": "µs" } ] } ] }, "gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 }, "id": 22, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "name" } ] }, "pluginVersion": "9.2.2", "targets": [ { "datasource": "iguazio", "hide": false, "rawQuery": true, "refId": "A", "target": "backend=kv;\ncontainer=users;\ntable=pipelines/$PROJECT/model-endpoints/endpoints;\nfilter=uid==\"$MODELENDPOINT\";\nfields=uid,model,function_uri,model_class,first_request,metrics,last_request;", "type": "table" } ], "transformations": [ { "id": "extractFields", "options": { "source": "metrics" } }, { "id": "extractFields", "options": { "source": "generic" } }, { "id": "organize", "options": { "excludeByName": { "generic": true, "latency_avg_5m": true, "metrics": true, "predictions_count_1h": true, "predictions_count_5m": true }, "indexByName": {}, "renameByName": { "endpoint_id": "Endpoint ID", "first_request": "First Request", "function": "Function", "function_uri": "Function URI", "last_request": "Last Request", "latency_avg_1h": "Average Latency (1 hour)", "latency_avg_1s": "Average Latency", "latency_avg_5m": "", "metrics": "", "model": "Model", "model_class": "Model Class", "predictions_per_second": "Predictions/s (5 minute avg)", "predictions_per_second_count_1s": "Predictions/sec", "tag": "Tag", "uid": "Endpoint ID" } } } ], "transparent": true, "type": "table" }, { "datasource": "iguazio", "description": "", "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "auto", "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "tvd_sum" }, "properties": [ { "id": "displayName", "value": "TVD (sum)" }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "tvd_mean" }, "properties": [ { "id": "displayName", "value": "TVD (mean)" }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "hellinger_sum" }, "properties": [ { "id": "displayName", "value": "Hellinger (sum)" }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "hellinger_mean" }, "properties": [ { "id": "displayName", "value": "Hellinger (mean)" }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "kld_sum" }, "properties": [ { "id": "displayName", "value": "KLD (sum)" }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "kld_mean" }, "properties": [ { "id": "displayName", "value": "KLD (mean)" }, { "id": "custom.align", "value": "center" } ] }, { "matcher": { "id": "byName", "options": "drift_measures" }, "properties": [ { "id": "custom.hidden", "value": false }, { "id": "mappings", "value": [] } ] } ] }, "gridPos": { "h": 3, "w": 24, "x": 0, "y": 3 }, "id": 25, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "name" } ] }, "pluginVersion": "9.2.2", "targets": [ { "datasource": "iguazio", "hide": false, "rawQuery": true, "refId": "A", "target": "backend=kv;\ncontainer=users;\ntable=pipelines/$PROJECT/model-endpoints/endpoints;\nfilter=uid==\"$MODELENDPOINT\";\nfields=drift_measures;", "type": "table" } ], "title": "Overall Drift Analysis", "transformations": [ { "id": "extractFields", "options": { "format": "json", "replace": false, "source": "drift_measures" } }, { "id": "filterFieldsByName", "options": { "include": { "names": [ "tvd_sum", "tvd_mean", "hellinger_sum", "hellinger_mean", "kld_sum", "kld_mean" ] } } } ], "transparent": true, "type": "table" }, { "datasource": "iguazio", "description": "Feature analysis of the latest batch", "fieldConfig": { "defaults": { "custom": { "align": "center", "displayMode": "auto", "filterable": false, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Expected Min" }, "properties": [ { "id": "noValue", "value": "N/A" } ] }, { "matcher": { "id": "byName", "options": "Expected Mean" }, "properties": [ { "id": "noValue", "value": "N/A" } ] }, { "matcher": { "id": "byName", "options": "Expected Max" }, "properties": [ { "id": "noValue", "value": "N/A" } ] }, { "matcher": { "id": "byName", "options": "tvd" }, "properties": [ { "id": "displayName", "value": "TVD" } ] }, { "matcher": { "id": "byName", "options": "hellinger" }, "properties": [ { "id": "displayName", "value": "Hellinger" } ] }, { "matcher": { "id": "byName", "options": "kld" }, "properties": [ { "id": "displayName", "value": "KLD" } ] } ] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 6 }, "id": 29, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "current_stats" } ] }, "pluginVersion": "9.2.2", "targets": [ { "datasource": "iguazio", "rawQuery": true, "refId": "A", "target": "backend=kv;\ncontainer=users;\ntable=pipelines/$PROJECT/model-endpoints/endpoints;\nfilter=uid==\"$MODELENDPOINT\";\nfields= current_stats;", "type": "table" }, { "datasource": "iguazio", "hide": false, "refId": "B", "target": "backend=kv; container=users; table=pipelines/$PROJECT/model-endpoints/endpoints; filter=uid==\"$MODELENDPOINT\"; fields= feature_stats;", "type": "table" }, { "datasource": "iguazio", "hide": false, "refId": "C", "target": "backend=kv; container=users; table=pipelines/$PROJECT/model-endpoints/endpoints; filter=uid==\"$MODELENDPOINT\"; fields= drift_measures;", "type": "table" } ], "title": "Features Analysis", "transformations": [ { "id": "extractFields", "options": { "format": "auto", "replace": false, "source": "current_stats" } }, { "id": "extractFields", "options": { "format": "auto", "source": "feature_stats" } }, { "id": "extractFields", "options": { "replace": false, "source": "drift_measures" } }, { "id": "merge", "options": {} }, { "id": "reduce", "options": { "includeTimeField": false, "labelsToFields": false, "mode": "seriesToRows", "reducers": [ "allValues" ] } }, { "id": "filterByValue", "options": { "filters": [ { "config": { "id": "equal", "options": { "value": "feature_stats" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "current_stats" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "timestamp" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "drift_measures" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "kld_sum" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "kld_mean" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "tvd_mean" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "tvd_sum" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "hellinger_sum" } }, "fieldName": "Field" }, { "config": { "id": "equal", "options": { "value": "hellinger_mean" } }, "fieldName": "Field" } ], "match": "any", "type": "exclude" } }, { "id": "extractFields", "options": { "replace": false, "source": "All values" } }, { "id": "filterFieldsByName", "options": { "include": { "names": [ "Field", "0", "1", "2" ] } } }, { "id": "extractFields", "options": { "replace": false, "source": "0" } }, { "id": "filterByValue", "options": { "filters": [ { "config": { "id": "isNull", "options": {} }, "fieldName": "1" }, { "config": { "id": "greater", "options": { "value": 0 } }, "fieldName": "2" } ], "match": "any", "type": "exclude" } }, { "id": "extractFields", "options": { "format": "json", "source": "1" } }, { "id": "extractFields", "options": { "source": "2" } }, { "id": "filterFieldsByName", "options": { "include": { "names": [ "Field", "mean 1", "min 1", "max 1", "mean 2", "min 2", "max 2", "tvd", "hellinger", "kld" ] } } }, { "id": "renameByRegex", "options": { "regex": "mean 1", "renamePattern": "Actual Mean" } }, { "id": "renameByRegex", "options": { "regex": "min 1", "renamePattern": "Actual Min" } }, { "id": "renameByRegex", "options": { "regex": "max 1", "renamePattern": "Actual Max" } }, { "id": "renameByRegex", "options": { "regex": "mean 2", "renamePattern": "Expected Mean" } }, { "id": "renameByRegex", "options": { "regex": "min 2", "renamePattern": "Expected Min" } }, { "id": "renameByRegex", "options": { "regex": "max 2", "renamePattern": "Expected Max" } }, { "id": "renameByRegex", "options": { "regex": "tvd", "renamePattern": "TVD" } }, { "id": "renameByRegex", "options": { "regex": "hellinger", "renamePattern": "Hellinger" } }, { "id": "renameByRegex", "options": { "regex": "kld", "renamePattern": "KLD" } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Actual Max": 6, "Actual Mean": 2, "Actual Min": 4, "Expected Max": 5, "Expected Mean": 1, "Expected Min": 3, "Field": 0, "Hellinger": 8, "KLD": 9, "TVD": 7 }, "renameByName": {} } } ], "transparent": true, "type": "table" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "iguazio", "fill": 1, "fillGradient": 1, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 16, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.2.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": "iguazio", "rawQuery": true, "refId": "A", "target": "backend=tsdb;\ncontainer=users;\ntable=pipelines/$PROJECT/model-endpoints/events;\nfilter=endpoint_id=='$MODELENDPOINT' AND record_type=='endpoint_features';", "type": "timeserie" } ], "thresholds": [], "timeRegions": [], "title": "Incoming Features", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": true, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "refresh": "30s", "schemaVersion": 37, "style": "dark", "tags": [], "templating": { "list": [ { "current": {}, "datasource": "model-monitoring", "definition": "target_endpoint=list_projects", "hide": 0, "includeAll": false, "label": "Project", "multi": false, "name": "PROJECT", "options": [], "query": "target_endpoint=list_projects", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": {}, "datasource": "iguazio", "definition": "backend=kv;\ncontainer=users;\ntable=pipelines/$PROJECT/model-endpoints/endpoints;\nfields=uid;", "hide": 0, "includeAll": false, "label": "Model Endpoint", "multi": false, "name": "MODELENDPOINT", "options": [], "query": "backend=kv;\ncontainer=users;\ntable=pipelines/$PROJECT/model-endpoints/endpoints;\nfields=uid;", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Model Monitoring - Details", "uid": "AohIXhAMk", "version": 9, "weekStart": "" }