delete obsolete documentation files

-- Select number of http error codes in the last hour

SELECT http_status, count(http_status) from http_requests

  WHERE time > -24hours

  GROUP BY http_status;

-- AGGREGATE_LOCKSTEP;

-- Select 90th percentile latency in the last day

SELECT percentile(latency, 90) from latency_metric WHERE time > -24hours;

-- SCAN_LOCKSTEP

-- SELECT error rate with two metrics and a 5 min window

SELECT

  time,

  ((delta(succesful_requests.count) / delta(errors.count)) * 100) as error_rate,

  FROM successful_requests, errors

  WHERE time > -24hours

  GROUP BY TIME_WINDOW(5 minutes);

-- AGGREGATE_TIME_WINDOW

-- SELECT top 10 slowest pages yesterday by 90th percentile latency

SELECT

  url, percentile(90, latency) as 90thpercentile_latency

  FROM request_log

  WHERE time > 24hours

  GROUP BY url

  ORDER BY 90thpercentile_latency DESC

-- SELECT all pages yesterday with a 90th percentile latency > 1000ms

SELECT

  url, percentile(90, latency) as 90thpercentile_latency

  FROM request_log

  WHERE time > 24hours

  GROUP BY url

  HAVING 90thpercentile_latency > 1000;

-- SELECT 90thpercentile page latency for three pages in 5 min windows

SELECT

  time, url, percentile(90, latency) as 90thpercentile_latency

  FROM request_log

  WHERE time > 24hours

  WHERE url IN ("/mypage1", "/mypage2", "/mypage3")

  GROUP BY TIME_WINDOW(5minutes), url;

-- COUNT number of events per time period

SELECT count(*) FROM metric GROUP BY TIME_WINDOW(5minutes);

-- SELECT holtwinters forecast for a metric value

SELECT real_value, holtwinters_forecast(real_value)

  FROM metric

  WHERE time > -24hours AND time < +24hours;

### More examples:

  -- error rate computed from joining the first derivates of two metrics over a

  -- moving 60 second window in the last hour

  SELECT

    "http error rate" as series,

    num_200.time as x,

    num_500.val / num_200.val as y

  FROM (

      SELECT time, delta(value) AS val

        FROM http_status_codes

        SINCE -60minutes UNTIL now

        WHERE status_code = 200

        GROUP BY TIMEWINDOW(time, 60, 10);

    ) AS num_200

  JOIN (

      SELECT time, delta(value) AS val

        FROM http_status_codes

        SINCE -60minutes UNTIL now

        WHERE status_code = 5200

        GROUP BY TIMEWINDOW(time, 60, 10);

    ) AS num_500

  ) ON num_200.time = num_500.time;

  -- error rate computed from joining the first derivates of two metrics over a

  -- moving 60 second window in the last hour

  SELECT

    "http error rate" as series,

    num_200.time as x,

    num_500.val / num_200.val as y

  FROM (

      SELECT time, delta(value) AS val

        FROM http_status_codes

        SINCE -60minutes UNTIL now

        WHERE status_code = 200

        GROUP BY TIMEWINDOW(time, 60, 10);

    ) AS num_200, (

      SELECT time, delta(value) AS val

        FROM http_status_codes

        SINCE -60minutes UNTIL now

        WHERE status_code = 5200

        GROUP BY TIMEWINDOW(time, 60, 10);

    ) AS num_500

  WHERE num_200.time = num_500.time;

-- insert distribution as map of $bucket -> $count pairs. expand each $bucket to

-- $count rows with value = $bucket on query

-- e.g. insert value {10:2, 100: 4, 250: 1}

-- select...

-- time, value

-- t0, 10

-- t0, 10

-- t0, 100

-- t0, 100

-- t0, 100

-- t0, 100

-- t0, 250

-- display the last hour of measurements

SELECT "mymetric" as series, time as x, value as y FROM mymetric,

-- display the moving average of our measurement over a moving 60s window

SELECT time as x, average(value) as FROM mymetric GROUP BY TIMEWINDOW(time 60, 10);

-- insert rationals/fractions (e.g. error rate)

-- allows proper aggregation over error rate:

-- display the aggregate error rate with a moving 60s window in the last hour

SELECT time as x, sum(numerator(value)) / sum(denominator(value)) as y

  FROM mymetric

  GROUP BY TIMEWINDOW(time, 60, 10)

  WHERE time > -60mins;

-- display the error rate per host with a moving 60s window in the last hour

SELECT time as x, sum(numerator(value)) / sum(denominator(value)) as y

  FROM mymetric

  GROUP BY TIMEWINDOW(time, 60, 10), hostname

  WHERE time > -60mins;

-- display the first derivative of our measurement over a moving 60s window in the last hour

SELECT time as x, delta(value) as y

  FROM mymetric

  GROUP BY TIMEWINDOW(time, 60, 10)

  WHERE time > -60mins;

-- ....equivalent to

SELECT time as x, delta(value) as y

  FROM mymetric

  GROUP BY TIMEWINDOW(time, 60, 10)

  SINCE -60mins UNTIL now;

-- number of requests per http status code over a moving 60 second window in

-- the last hour

SELECT status_code as series, time as x, delta(value) as y

  FROM http_status_codes

  GROUP BY TIMEWINDOW(time, 60, 10), status_code;

-- error rate computed from joining two metrics over a moving 60 second window

-- in the last hour

SELECT

  hostname as series,

  time as x,

  delta(error_metric.value) / delta(success_metric.value) as y

FROM

  success_metric,

  error_metric

WHERE

  time > -60minutes

GROUP BY

  TIMEWINDOW(time, 60, 10),

  hostname;

Another example:

Draw a few latency distributions for multiple loadtest experiments sourced from

multiple csv files. The csv files contain two columns "time" and "runtime" and

one line for each request in the load test. The "time" column contains the wall

clock time at which the request was issued and the "runtime" column contains the

runtime of the request.

We will draw a single line graph that plots requests per minute vs the 90th

percentile latency in that time window.

    IMPORT experiment1 FROM "experiment_1.csv";

    DRAW LINE CHART;

    SERIES "experiment1" FROM

      SELECT count(*) as qps, nth_percentile(runtime, 90) as latency

        FROM experiment1

        GROUP BY time_window(time, 1s);

Lets add another experiment as a second series in the chart:

    IMPORT experiment1 FROM "experiment_1.csv";

    IMPORT experiment2 FROM "experiment_2.csv";

    BEGIN LINES;

    CREATE SERIES WITH SELECT

      "experiment1" as title, count(*) as x, nth_percentile(runtime, 90) as y

      FROM experiment1

      GROUP BY time_window(time, 1s);

    SERIES "experiment2" FROM

      SELECT count(*) as qps, nth_percentile(runtime, 90) as latency

        FROM experiment2

        GROUP BY time_window(time, 1s);

To make that last query more interesting lets say we put all our test results

into a single file with an additional "experiment" columns and try to recreate

the same chart:

    -- load a csv file with per request logs from a load test

    IMPORT loadtests FROM "myfile.csv";

    -- draw a graph of qps vs 90thpercentile, one series for each experiment

    BEGIN CHART;

    DRAW LINES experiment FROM

      SELECT experiment, count(*) as qps, nth_percentile(runtime) as latency

        FROM loadtests

        GROUP BY experiment, time_window(time, 1s);

[ release ]

    - hover flyoouts for button bar in sql editor "Refresh, Change View and use CTRL+enter"

    - median

    - fix mean

-----

[ faq ]

    - how do i set the line color

    - can i use fnordmetric with external data

    - do I need an external database

    - FM legt die timestamps der samples _immer_ selbst bei dateneingang fest?

[ feature q ]

    - compare with yesterday in metric preview widget

    - refresh btn in metric preview widget

    - round fn

    - percentiles

    - metric preview widget: auto refresh (on/off)

    - version numbers

    - deb pkg

    - homebrew pkg

    - doc: interactive charts

    - ruby client + doc

    - javascript client + doc

    - make query editor split draggable/resizable

    - functions: delta

    - vertical hover line in chartextensions

    - sql time helpers (1hour, etc)

    - support red/green/blue colors

    - max threads in takscheduler?

    - functions: variance, stddev, nth_derivate,

    - union

    - cross join

    - GROUP OVER TIMEWINDOW on multiple tables

    - find time constraint from sql query

    - scan samples with range / sstable binary search

    - sql: in

    - sql: if

    - sql: case

    - dashboard params

    - dashboards: Wanted feature: publish static to S3

    - retention / downsampling + DOC

    - merge compaction

    - SQL: having

    - SQL: cross joins

    - SQL: subqueries

    - better handling of missing/NULL values! in seriesadapter

    - mysql: re-serialize where condition

    - mysql backend

    - hbase backend

    - SQL: JOIN

    - ui: line/point styles

    - impl inside/outside axis labels

    - impl rotated axis labels

    - series adapter: max interpolation gap

    - middle axis

    - impl variable axis label size

    - make # of axis ticks configurable via sql

    - set option: date format string

    - set option: timezone

    - set option: number format (si, exp, etc)

    - chart type: histogram

    - chart type: heatmap

    - ui: stacked area charts

    - mmap cache for readonly files

    - dashboard hosting (--dashboards <dir>)

    - alerts

    - events (annotations)

    - chart type: ganttchart

    - chart type: boxplot

    - chart type: error bars

    - chart type: pie chart

    - sparklines

    - postgresql backend

    - smooth paths

    - query cache

    - holtwinters

    - dashboard edit ui: html source

    - dashboard edit ui: wysiwyg

    - candlestick

    - sql: query plan optimizer. push limit/offset/order down into tablescan if

           no complex nodes inbetween

    - report with markdown + chartsql

[ bug q ]

    - svg: escape series names, labels!

    - multiple unnamed select/series should not be merged

    - bug: NAN is not a valid value in SVG

    - SELECT count(*) AS row_count FROM `/osx/load_avg_5m`; --- does not return col name

    - bug: line chart with only negative values

    - bug: 2d area chart with explicit y domain, min > 0

    - proper last insert on restart

[ refactor q ]

    - refactor nextRow(SValue* row, size_t len) -> nextRow(RowType) (std::vector<SValue>)

    - refactor tokenizer // dynamically register tokens

    - rename (axis|legend|grid)definition -> \1config

Getting Started with ChartSQL

=============================

_This guide will walk you through the required steps of executing a simple ChartSQL query 

from the command line. If you haven't installed FnordMetric yet, start by reading the

[Installation page](/documentation/installation) ._

For our first simple example we will create a line chart that displays the

average temperatures of multiple cities by month resulting in the following output:

<img src="/img/simple_linechart.png" width="800" />

Importing data from CSV

-----------------------

ChartSQL allows you to import tables from external data sources using the

IMPORT statement. Assume we have a CSV file called city_temperatures.csv 

where the content looks similar to this:

    city,month,temperature

    Tokyo,Dec,9.6

    New York,Nov,8.6

    ...

Download city_temperatures.csv to a folder on your system. Within the same folder create another file called example_query.sql with the following content:

    IMPORT TABLE city_temperatures FROM "csv://city_temperatures.csv?headers=true"

    SELECT * FROM city_temperatures;

This will import the table `city_temperatures` with three columns `city`, `month`

and `temperature` from our csv file and return each of its rows.

In order to execute the query we just wrote from the command line, we use the

fnordmetric-cli binary:

    $ fnordmetric-cli example_query.sql

Doing so should give an output similar to the following:

    ==================================

    | city   | month  | temperature  |

    ==================================

    | Tokyo  | Dec    | 9.6          |

    | Tokyo  | Nov    | 13.9         |

    | Tokyo  | Oct    | 18.3         |

    | Tokyo  | Sep    | 23.3         |

    | ...    | ...    | ...          |

Visualizing SQL Query Results

------------------------------

Now lets create a simple line chart from the data. ChartSQL's DRAW statement

allows you to specify that the query result should be returned as a chart (or

other visualization) rather than a table.

All SELECT statements that follow a DRAW statement are interpreted as chart data.

and must return at least a `x` and a `y` result list column. In our example

we want to plot the months on the X axis and the corresponding temperatures on the Y axis,

so we add this DRAW statement to our example_query.sql file:

    IMPORT TABLE city_temperatures FROM "csv://city_temperatures.csv?headers=true"

    DRAW LINECHART;

    SELECT city as series, month as X, temperature AS Y FROM city_temperatures;

Let's execute the query and render the chart to SVG:

    $ fnordmetric-cli --format svg example_query.sql > example_chart.svg

Now open example_chart.svg in a browser. You'll see a chart that looks

somewhat like the chart in the beginning of this guide. We are still missing

axes, the legend and points. After adding these, we have completed our

example_query.sql file:

    IMPORT TABLE city_temperatures FROM "csv://city_temperatures.csv?headers=true"

    DRAW LINECHART WITH

        AXIS LEFT

        AXIS BOTTOM

        LEGEND TOP LEFT INSIDE;

    SELECT

        city as series,

        month as X,

        temperature AS Y,

        "circle" as pointstyle

    FROM city_temperatures;

You are now able to execute a simple ChartSQL query from the command line,

but there is a lot more to discover! These are some good starting points for further reading:

  + [Examples](/examples/)

  + [The ChartSQL Query Language](/documentation/chartsql/introduction/)

  + [Getting Started with FnordMetric Server](/documentation/getting_started/fnordmetric-server/)

Getting Started with FnordMetric Server

=======================================

_This guide will walk you through starting a FnordMetric Server instance, inserting

metric data and querying the metric data with ChartSQL. If you do not have installed

FnordMetric yet, read the [Installation page](/documentation/installation) first._

Fnordmetric Server is a standalone HTTP server application. It exposes a web UI

and a HTTP API to run ChartSQL queries and collect timeseries data.

You can start FnordMetric Server with or without a "storage backend".

If FnordMetric Server is started without a storage backend you can only use the

web interface to execute ChartSQL queries against external data sources (like a

MySQL database). If it is started with a storage backend, you can also use the

HTTP (and optionally the statsd) API to collect timeseries data into the storage

backend and subsequently query that timerseries data using ChartSQL.

FnordMetric Server currently supports three storage backends: `inmemory`, `disk`,

and `hbase`.

#### Starting Fnordmetric Server

For the getting started guide we will use the `disk` backend which stores the

metric data in a folder on the local hard disk. To start a FnordMetric server

instance with a local disk storage backend on HTTP port 8080 run:

    $ mkdir -p /tmp/fnordmetric-data

    $ fnordmetric-server --http_port 8080 --statsd_port 8125 --storage_backend disk --datadir /tmp/fnordmetric-data

#### Collecting Timeseries Data

FnordMetric Server records timeseries data in "Metrics". A Metric is somewhat

equivalent to a table in a regular SQL database. Each metric has a unique name

and consists of a collection of data points called "samples" that are recorded

over time (i.e. a timeseries).

A "sample" is a single datapoint. Each sample contains at least a timestamp

and a numeric value. To keep the table analogy, each metric is a table that has

two default columns `value` and `time` and each sample is a row in that table.

You can query metrics using ChartSQL like normal tables:

    > select time, value from mymetric;

    ===================================

    | time                  |  value  |

    ===================================

    | 2014-11-08 20:30:12   |  0.913  |

    | 2014-11-08 20:30:42   |  0.837  |

    | 2014-11-08 20:31:13   |  0.638  |

    | 2014-11-08 20:31:41   |  0.326  |

    | ...                   |  ...    |

As an example, we will monitor the http response times of a fictional web

application. We will create one "metric" that will be called `http\_response\_times`

and will record the http response times (latencies) of our application. We are

going to insert a sample into this metric for each HTTP request that our web

application serves.

There are a number of client libraries that allow you to send samples to

FnordMetric Server using the HTTP or statsd API. For now, lets cheat a bit and

manually send samples from the command line. The simplest way to send samples

from your command line is using the statsd API. If you started FnordMetric Server

on port 8125 (see above), you can use the netcat utility to send a sample via

UDP+statsd.

Let's insert the value `42` into the `http\_response\_times` metric. In our example this

means we record the response time of a single HTTP request that took 42ms.

    $ echo "http_response_times:42" | nc -u -w0 127.0.0.1 8125

Execute this command a few times with different values to insert multiple samples

into the metric. FnordMetric Server will automatically create the metric if it

doesn't exist yet.

#### Execute Queries from the Web Interface

You should now be able to navigate to the admin interface on

`http://localhost:8080/` in your browser and see our newly created metric. Click

on the metric to bring up the interactive query editor. This should look something

like this:

<div style="width:960px; margin:50px auto 40px auto; height:583px; overflow: hidden;" class="shadow">

  <video width="960" height="583" autoplay="autoplay" loop>

    <source src="/fnordmetric-server.mp4" type="video/mp4">

    <script type="text/javascript">

      if (!(navigator.userAgent.search("Safari") >= 0 && navigator.userAgent.search("Chrome") < 0)) {

        document.write('<source src="/fnordmetric-server.webm" type="video/webm">');

      }

    </script>

    <img src="/img/fnordmetric_server_screen.png" width="875" />

  </video>

</div>

Click around a bit to make yourself familiar with the Web UI. The charts you see

above are generated using ChartSQL. The queries are  automatically generated by

the interactive query editor. Here is an example query:

<i>Display the 100 latest samples from `http\_response\_times` as a line chart with the

sample time plotted on the X axis and the sample value plotted on the Y axis.:</i>

    DRAW LINECHART

        AXIS LEFT

        AXIS BOTTOM;

    SELECT time as x, value as y

        FROM http_response_times

        ORDER BY time DESC

        LIMIT 100;

#### Adding labels

To allow you to drill down into your metric data in arbitrary dimensions, each

sample can optionally be labelled with one or more "labels". Each label is a

key: value pair.

In our example, assume we run our web application on multiple hosts in different

datacenters. It would be nice to label each sample with  `hostname=...` and

`datacenter=...` so that we can roll up the http response times by host, datacenter

or a combinaton of both.

Let's insert a few more example samples into our <code>http\_response\_times</code> metric

and attach these labels:

    $ echo "http_response_times[hostname=machine82][datacenter=ams1]:18" | nc -u -w0 127.0.0.1 8125

    $ echo "http_response_times[hostname=machine83][datacenter=ams1]:42" | nc -u -w0 127.0.0.1 8125

    $ echo "http_response_times[hostname=machine84][datacenter=ams1]:23" | nc -u -w0 127.0.0.1 8125

When querying metrics with ChartSQL, the label keys act as table columns so you

can filter and aggregate/group by label values. Our <code>http\_response\_times</code> table

now has 4 columns:

    > select time, value, hostname, datacenter from http_response_times;

    ==============================================================

    | time                  | value  | hostname   | datacenter  |

    ==============================================================

    | 2014-11-08 20:30:12   | 18     | machine82  | ams1        |

    | 2014-11-08 20:30:12   | 42     | machine83  | ams1        |

    | 2014-11-08 20:30:12   | 23     | machine84  | ams1        |

    | ...                   | ...    | ...        | ...         |

You can execute this query from the interactive query editor to display the last

hour of samples in the `http\_response\_times` metric rolled up by hostname. It

will draw a line chart with the sample time plotted on the X axis and the sample value

plotted on the Y axis and one series per hostname:

    DRAW LINECHART

        AXIS LEFT

        AXIS BOTTOM;

    SELECT hostname as series, time as x, value as y

        FROM http_response_times

        WHERE time > -1hour;

The result should look something like this:

<img src="/img/fnordmetric_server_screen2.png" width="800" class="shadow" />

<br />

You now have a running FnordMetric Server, but there is a lot more you can do.

These are good docs to read next:

  + [Examples](/examples/)

  + [ChartSQL Query Language](/documentation/chartsql/introduction/)

  + [Time-window aggregations](/documentation/chartsql/timewindow_aggregations/)

This guide will walk you through building a simple HTML5 dashboard using ChartSQL

and FnordMetric Server:

  + [Gettting Started with HTML5 Dashboards in 5 Minutes](/documentation/html5_dashboards)

FnordMetric HTML5 API Reference

===============================

_The HTML5 API is still a Work In Progress — Pull Requests welcome!_

### Introduction

The FnordMetric UI HTML5 API allows you to plugin real-time data and

charts into any website without having to write any JavaScript code. This is

achieved by including a JavaScript library and using the FnordMetric HTML5

components.

The JavaScript library `fnordmetric.js` is bundled with FnordMetric Server, to

include it put this line into your HTML file:

    <!-- include fnordmetric.js from fnordmetric-server-host:port -->

    <script src='http://fnordmetric-server-host:port/s/fnordmetric.js' type='text/javascript'></script>

    <!-- example: include fnordmetric.js from localhost:8080 -->

    <script src='http://localhost:8080/s/fnordmetric.js' type='text/javascript'></script>

### Component: \<fm-chart\>

The `\<fm-chart\>` tag executes a ChartSQL query and displays the result as a chart.

For Example:

    <fm-chart style="width: 800px;">

      DRAW LINECHART

          AXIS BOTTOM;

      SELECT

          time AS x, value AS y FROM ...;

    </fm-chart>

These are the valid html attributes for `\<fm-chart\>` tags:

<table>

  <tr>

    <th>data-autoupdate</th>

    <td>

      refresh the chart every N seconds

    </td>

  </tr>

</table>