Inside Pandora

2016.06.08

Jesse Fang

Architecture

ElasticSearch

Build on Apache Lucene, open source, 16855 stars in GitHub
Distributed, scalable, and highly available
Near-Realtime full-text search
Document-oriented & schema free
RESTful API & structured query DSL
Elastic Stack

Setup ElasticSearch in 5 mins

ELK in Pandora

Web Log

[2016-06-06 22:08:07.833] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile"],"keyword":"*","page":"0"}, IsNoResult:false, Time:312
[2016-06-06 22:10:09.938] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile"],"keyword":"klahanie","page":"0"}, IsNoResult:false, Time:344
[2016-06-06 23:07:10.355] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile","cortana"],"keyword":"broswer.main","page":"0"}, IsNoResult:true, Time:484
[2016-06-07 12:53:10.442] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile","cortana"],"keyword":"anid","page":"0"}, IsNoResult:false, Time:1375
[2016-06-07 12:55:00.352] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile","cortana"],"keyword":"Events.UserInfo.Anid","page":"0"}, IsNoResult:false, Time:13153
[2016-06-07 17:18:18.047] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile"],"keyword":"ie log generator","page":"0"}, IsNoResult:false, Time:2002
[2016-06-07 17:19:44.427] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile"],"keyword":"oculus vr","page":"0"}, IsNoResult:false, Time:937
[2016-06-07 20:16:47.373] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile","cortana"],"keyword":"Page_ClickCount","page":"0"}, IsNoResult:false, Time:14299
[2016-06-07 20:17:05.630] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile","cortana"],"keyword":"Page_ClickCount=","page":"0"}, IsNoResult:true, Time:469
[2016-06-07 20:17:17.524] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile","cortana"],"keyword":"Page_ClickCount>0","page":"0"}, IsNoResult:false, Time:610
[2016-06-07 20:20:15.951] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile","cortana"],"keyword":"Page_ClickCount>0","page":"0"}, IsNoResult:false, Time:376
[2016-06-07 20:24:25.036] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile","cortana"],"keyword":"Page_ClickCount > 1","page":"0"}, IsNoResult:false, Time:1368
[2016-06-07 23:08:59.906] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile"],"keyword":"OneBox","page":"0"}, IsNoResult:false, Time:2609
[2016-06-07 23:09:23.342] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile"],"keyword":"Page.OneBox","page":"0"}, IsNoResult:true, Time:312
[2016-06-07 23:09:29.751] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":["bing","mobile"],"keyword":"*","page":"0"}, IsNoResult:false, Time:1593
[2016-06-07 23:10:07.911] [INFO] query-logger - UNKNOWN, Params:{"abnormal":"false","autosuggest":"false","dataset":"bing","keyword":"*","page":"0"}, IsNoResult:false, Time:110

Logstash Config

input {
    file {
        type => "query"
        path => "D:/Pandora/logs/pandora-query.log"
        start_position => beginning 
        ignore_older => 0
    }
}

filter {
    if [type] == "query" {
        grok {
            match => { "message" => "\[%{TIMESTAMP_ISO8601:logtimeString}\] \[%{LOGLEVEL:level}\] query-logger - (%{USERNAME:domain}\\)?%{USERNAME:alias}, Params:%{GREEDYDATA:params}, IsNoResult:%{WORD:isNoResult}, Time:%{NONNEGINT:duration:int}"}
        }
        json {
            source => "params"
        }
        date {
            match => ["logtimeString", "YYYY-MM-dd HH:mm:ss.SSS"]
            target => "logtime"
        }
    }
}

output {
    stdout {}
    if [type] == "query" {
        elasticsearch {
            hosts => "lsstc451"
            # NOTE: index name must be lower case
            index => "pandora_query_log-%{+YYYY}"
        }
    }
}