Skip to content

Commit

Permalink
Initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@ScarletTanager
ScarletTanager committed Aug 10, 2017
0 parents commit 1c5cf5b
Show file tree
Hide file tree
Showing 11 changed files with 1,082 additions and 0 deletions.
13 changes: 13 additions & 0 deletions ccruncher_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package ccruncher_test

import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

"testing"
)

func TestCcruncher(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Ccruncher Suite")
}
69 changes: 69 additions & 0 deletions cmd/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package main

import (
"ccruncher"
"fmt"
"os"

"github.com/cloudfoundry-incubator/candiedyaml"
)

type App struct {
GUID string
Requests []Request
}

type Request struct {
RequestID string
LogEntries []ccruncher.LogEntry
}

func main() {
if len(os.Args) < 2 {
os.Exit(1)
}

log, err := os.Open(os.Args[1])

if err != nil {
fmt.Fprintln(os.Stderr, "Could not open file ", os.Args[1])
os.Exit(1)
}

ccLog, _ := ccruncher.ParseLog(log)

outFile, err := os.Create("ccLog.yml")

if err != nil {
os.Exit(2)
}

defer outFile.Close()

appGuids := ccLog.Apps()

for _, guid := range appGuids {
if guid != "unspecified" {
app := &App{
GUID: guid,
}
ids := ccLog.RequestsForApp(guid)

for _, id := range ids {
app.Requests = append(app.Requests, Request{
RequestID: id,
LogEntries: ccLog.EntriesForRequest(id),
})

e, _ := candiedyaml.Marshal(app)
outFile.Write(e)
// entries := ccLog.EntriesForRequest(id)
//
// for _, entry := range entries {
// e, _ := entry.Render()
// outFile.Write(e)
// }
}
}
}
}
41 changes: 41 additions & 0 deletions fixtures/badlog
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
begin 644 pork
M<&%C:V%G92!C8W)U;F-H97(*"FEM<&]R="`H"@DB8G5F:6\B"@DB96YC;V1I
M;F<O:G-O;B(*"2)F;70B"@DB:6\B"@DB<F5G97AP(@H)(G-T<FEN9W,B"BD*
M"G1Y<&4@3&]G16YT<GD@<W1R=6-T('L*"51I;65S=&%M<"`@9FQO870V-"!@
M:G-O;CHB+&]M:71E;7!T>2)@"@E-97-S86=E("`@('-T<FEN9R`@8&IS;VXZ
M(BQO;6ET96UP='DB8`H)3&]G3&5V96P@("!S=')I;F<@(&!J<V]N.B)L;V=?
M;&5V96PL;VUI=&5M<'1Y(F`*"5-O=7)C92`@("`@<W1R:6YG("!@:G-O;CHB
M+&]M:71E;7!T>2)@"@E$871A("`@("`@(&UA<%MS=')I;F==:6YT97)F86-E
M>WT*"49I;&4@("`@("`@<W1R:6YG(&!J<V]N.B(L;VUI=&5M<'1Y(F`*"4QI
M;F5.=6UB97(@:6YT("`@(&!J<V]N.B)L:6YE;F\L;VUI=&5M<'1Y(F`*"4UE
M=&AO9"`@("`@<W1R:6YG(&!J<V]N.B(L;VUI=&5M<'1Y(F`*?0H*8V]N<W0@
M=F-A<%)E<75E<W1)1%)E9V5X(#T@(G9C87`M<F5Q=65S="UI9#H@6UQ<=UQ<
M+5TK(@H*9G5N8R!087)S94QO9RAL;V=&:6QE(&EO+E)E861E<BD@*%M=3&]G
M16YT<GDL(&5R<F]R*2!["@EV87(@96YT<FEE<R!;74QO9T5N=')Y"@EV87(@
M96YT<GD@3&]G16YT<GD*"79A<B!L:6YE3G5M8F5R('5I;G0*"@ES8V%N;F5R
M(#H](&)U9FEO+DYE=U-C86YN97(H;&]G1FEL92D*"@EF;W(@<V-A;FYE<BY3
M8V%N*"D@>PH)"6QI;F5.=6UB97(K*PH*"0EE<G(@.CT@:G-O;BY5;FUA<G-H
M86PH6UUB>71E*'-C86YN97(N5&5X="@I*2P@)F5N=')Y*0H*"0EI9B!E<G(@
M(3T@;FEL('L*"0D)<F5T=7)N(&5N=')I97,L(&9M="Y%<G)O<F8H(E!A<G-E
M3&]G*"D@97)R;W(@870@;&EN92`E9#H@)7,B+"!L:6YE3G5M8F5R+"!E<G(I
M"@D)?0H*"0EE;G1R:65S(#T@87!P96YD*&5N=')I97,L(&5N=')Y*0H)?0H*
M"7)E='5R;B!E;G1R:65S+"!N:6P*?0H*9G5N8R!297%U97-T241&;W)!<'`H
M87!P1U5)1"!S=')I;F<L(&QO9W-L:6-E(%M=8GET92D@*'-T<FEN9RP@97)R
M;W(I('L*"65N=')Y+"!E<G(@.CT@<&%R<V5,;V=%;G1R>2AL;V=S;&EC92D*
M"@EI9B!E<G(@(3T@;FEL('L*"0ER971U<FX@(B(L(&5R<@H)?0H*"7)E='5R
M;B!R97%U97-T241&<F]M365S<V%G92AE;G1R>2Y-97-S86=E*0I]"@IF=6YC
M(')E<75E<W1)1$9R;VU-97-S86=E*&UE<W-A9V4@<W1R:6YG*2`H<W1R:6YG
M+"!E<G)O<BD@>PH)<G@L(&5R<B`Z/2!R96=E>'`N0V]M<&EL92AV8V%P4F5Q
M=65S=$E$4F5G97@I"@EI9B!E<G(@(3T@;FEL('L*"0ER971U<FX@(B(L(&9M
M="Y%<G)O<F8H(E)E9V5X<"!C;VUP:6QA=&EO;B!F86EL=7)E.B`E<R(L(&5R
M<BD*"7T*"@EV87(@9FER<W1-871C:"!S=')I;F<*"@EI9B!F:7)S=$UA=&-H
M(#T@<G@N1FEN9%-T<FEN9RAM97-S86=E*3L@9FER<W1-871C:"`]/2`B(B![
M"@D)<F5T=7)N("(B+"!F;70N17)R;W)F*")5;F%B;&4@=&\@97AT<F%C="!V
M8V%P+7)E<75E<W0M:60@9G)O;2`E<R(L(&UE<W-A9V4I"@E]"@H)<F5T=7)N
M('-T<FEN9W,N4W!L:70H9FER<W1-871C:"P@(B`B*5LQ72P@;FEL"GT*"F9U
M;F,@<&%R<V5,;V=%;G1R>2AE;G1R>4I33TX@6UUB>71E*2`H3&]G16YT<GDL
M(&5R<F]R*2!["@EV87(@96YT<GD@3&]G16YT<GD*"65R<B`Z/2!J<V]N+E5N
M;6%R<VAA;"AE;G1R>4I33TXL("9E;G1R>2D*"@ER971U<FX@96YT<GDL(&5R
$<@I]"@``
`
end
300 changes: 300 additions & 0 deletions fixtures/cclog.1

Large diffs are not rendered by default.

300 changes: 300 additions & 0 deletions fixtures/cclog.2

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions fixtures/cclog.3
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"timestamp":1491835502.3272069,"message":"Completed 200 vcap-request-id: d462237c-c201-4ab2-57c3-e1e79570b0b0::0bfa387c-38df-4053-9dc9-3205d102ddf2","log_level":"info","source":"cc.api","data":{},"thread_id":47176176436100,"fiber_id":47176144229040,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":24,"method":"call"}
6 changes: 6 additions & 0 deletions fixtures/cclog.4
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{"timestamp":1491835502.212968,"message":"Started GET \"/v2/apps/3b664a4a-3aba-4a1b-baa7-8ab67b4ade96/stats\" for user: oeruntime_admin, ip: 75.126.23.243 with vcap-request-id: d462237c-c201-4ab2-57c3-e1e79570b0b0::0bfa387c-38df-4053-9dc9-3205d102ddf2 at 2017-04-10 14:45:02 UTC","log_level":"info","source":"cc.api","data":{},"thread_id":47176176436100,"fiber_id":47176144229040,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":12,"method":"call"}
{"timestamp":1491835502.4901989,"message":"Started GET \"/v2/apps/3a1e444a-6d92-4bc8-8579-f976f57b1239\" for user: 58e4f12b-d546-426a-908e-8f2ba15d2808, ip: 169.55.202.148 with vcap-request-id: c7c1cb2c-c09f-48f4-484b-ff665c5794b3::c6fe45c4-e20c-4976-9095-822216215d5f at 2017-04-10 14:45:02 UTC","log_level":"info","source":"cc.api","data":{},"thread_id":47176176368980,"fiber_id":47176139407000,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":12,"method":"call"}
{"timestamp":1491835502.7958727,"message":"Started GET \"/v2/apps/b8b1ff95-fa6a-4b25-933f-c69e0e39243c/stats\" for user: oeruntime_admin, ip: 169.45.162.88 with vcap-request-id: 9c09bb41-0f10-41f2-43a8-c60689968aba::e19732c7-c827-42e9-81fd-89a61f2b4157 at 2017-04-10 14:45:02 UTC","log_level":"info","source":"cc.api","data":{},"thread_id":47176176435800,"fiber_id":47176157122140,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":12,"method":"call"}
{"timestamp":1491835502.9892302,"message":"Started GET \"/v2/apps/1b86d700-1c7f-4479-b232-d677e85131b2?inline-relations-depth=1\" for user: elkadapter, ip: 169.55.202.148 with vcap-request-id: 6546bc4b-b7b4-4609-404d-ad642baf9488::77061278-2110-4b62-8791-d9a3bfe5200d at 2017-04-10 14:45:02 UTC","log_level":"info","source":"cc.api","data":{},"thread_id":47176176436100,"fiber_id":47176144229040,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":12,"method":"call"}
{"timestamp":1491835503.0352755,"message":"Started GET \"/v2/apps/f923ffea-0b19-4799-8f8e-175837eeed6a\" for user: oeruntime_admin, ip: 169.45.162.88 with vcap-request-id: 25e15ac7-ea93-4f18-5e88-23b1f5759070::be8d8a6e-661f-4e11-9e96-f8fe43ccba33 at 2017-04-10 14:45:03 UTC","log_level":"info","source":"cc.api","data":{},"thread_id":47176176436840,"fiber_id":47176140357680,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":12,"method":"call"}
{"timestamp":1491835503.0493014,"message":"Started GET \"/v2/apps/19990ba2-d858-4df6-a6de-c1a31ee30730/routes\" for user: 788f99af-6faa-496e-8038-a917ad016c5b, ip: 75.126.37.75 with vcap-request-id: 988f5308-8970-4821-7a0a-ab2b1de4cf04::8436dd2d-f6a1-4998-b306-1f916b3ab246 at 2017-04-10 14:45:03 UTC","log_level":"info","source":"cc.api","data":{},"thread_id":47176176369540,"fiber_id":47176177962920,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":12,"method":"call"}
4 changes: 4 additions & 0 deletions fixtures/cclog.5
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{"timestamp":1491835502.9892302,"message":"Started GET \"/v2/apps/1b86d700-1c7f-4479-b232-d677e85131b2?inline-relations-depth=1\" for user: elkadapter, ip: 169.55.202.148 with vcap-request-id: 6546bc4b-b7b4-4609-404d-ad642baf9488::77061278-2110-4b62-8791-d9a3bfe5200d at 2017-04-10 14:45:02 UTC","log_level":"info","source":"cc.api","data":{},"thread_id":47176176436100,"fiber_id":47176144229040,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":12,"method":"call"}
{"timestamp":1491835502.991367,"message":"Started GET \"/v2/apps/1b86d700-1c7f-4479-b232-d677e85131b2?inline-relations-depth=1\" for user: elkadapter, ip: 75.126.23.243 with vcap-request-id: 9c5a10d7-f3ca-47a5-6bee-efb8dd422010::f18a7d92-61bf-47bf-ba7a-9a833977afd1 at 2017-04-10 14:45:02 UTC","log_level":"info","source":"cc.api","data":{},"thread_id":47176176435300,"fiber_id":47176155691320,"process_id":29833,"file":"/var/vcap/packages/cloud_controller_ng/cloud_controller_ng/middleware/request_logs.rb","lineno":12,"method":"call"}
{"timestamp":1491835502.9917338,"message":"cc.dispatch","log_level":"debug","source":"cc.api","data":{"request_guid":"6546bc4b-b7b4-4609-404d-ad642baf9488::77061278-2110-4b62-8791-d9a3bfe5200d","endpoint":"read","args":["1b86d700-1c7f-4479-b232-d677e85131b2"]},"thread_id":47176176436100,"fiber_id":47176144229040,"process_id":29833,"file":"/var/vcap/data/packages/cloud_controller_ng/13404df2f089744e094e0574e2d3c66843e23fc7.1-eb9ae4e411ad595852dcbf6781e0ea131db756b3/cloud_controller_ng/app/controllers/base/base_controller.rb","lineno":78,"method":"dispatch"}
{"timestamp":1491835502.9965017,"message":"cc.dispatch","log_level":"debug","source":"cc.api","data":{"request_guid":"9c5a10d7-f3ca-47a5-6bee-efb8dd422010::f18a7d92-61bf-47bf-ba7a-9a833977afd1","endpoint":"read","args":["1b86d700-1c7f-4479-b232-d677e85131b2"]},"thread_id":47176176435300,"fiber_id":47176155691320,"process_id":29833,"file":"/var/vcap/data/packages/cloud_controller_ng/13404df2f089744e094e0574e2d3c66843e23fc7.1-eb9ae4e411ad595852dcbf6781e0ea131db756b3/cloud_controller_ng/app/controllers/base/base_controller.rb","lineno":78,"method":"dispatch"}
202 changes: 202 additions & 0 deletions readlog.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
package ccruncher

import (
"bufio"
"encoding/json"
"fmt"
"io"
"regexp"
"strings"
)

type CCLog struct {
entries map[string][]LogEntry
requestIDs map[string][]string
}

type LogEntry struct {
Timestamp float64 `json:",omitempty"`
Message string `json:",omitempty"`
LogLevel string `json:"log_level,omitempty"`
Source string `json:",omitempty"`
Data *LogEntryData
File string `json:",omitempty"`
LineNumber int `json:"lineno,omitempty"`
Method string `json:",omitempty"`
Request *Request
}

type LogEntryData struct {
RequestGUID string `json:"request_guid,omitempty"`
ProcessGUID string `json:"process_guid,omitempty"`
}

type Request struct {
RequestID string
AppGUID string
Method string
URIPath string
}

func (e LogEntry) RequestID() string {
return e.Request.RequestID
}

func (e LogEntry) AppGUID() string {
return e.Request.AppGUID
}

func (e LogEntry) HttpMethod() string {
return e.Request.Method
}

func (e LogEntry) URIPath() string {
return e.Request.URIPath
}

const unspecifiedRequestID = "unspecified"
const unspecifiedAppGUID = "unspecified"
const vcapRequestIDRegex = "vcap-request-id: [\\w\\-\\:]+"
const appGUIDRegex = "\\/v2\\/apps/[\\w\\-]+"
const methodAndURIPathRegex = `Started [A-Z]+ \"[\w\/\-]+`

func (c *CCLog) EntriesForRequest(requestID string) []LogEntry {
return c.entries[requestID]
}

func (c *CCLog) Entries() []LogEntry {
var entries []LogEntry

// e is of type []LogEntry
for _, e := range c.entries {
entries = append(entries, e...)
}

return entries
}

func (c *CCLog) Apps() []string {
var guids []string

for guid, _ := range c.requestIDs {
guids = append(guids, guid)
}

return guids
}

func (c *CCLog) RequestsForApp(appGUID string) []string {
if ids, exists := c.requestIDs[appGUID]; exists == true {
return ids
}

return nil
}

func ParseLog(logFile io.Reader) (*CCLog, error) {
var ccLog *CCLog
var entry LogEntry
var lineNumber uint

ccLog = &CCLog{
entries: make(map[string][]LogEntry),
requestIDs: make(map[string][]string),
}

scanner := bufio.NewScanner(logFile)

for scanner.Scan() {
lineNumber++

entry.Data = &LogEntryData{}

err := json.Unmarshal([]byte(scanner.Text()), &entry)

if err != nil {
return ccLog, fmt.Errorf("ParseLog() error at line %d: %s", lineNumber, err)
}

id, _ := requestIDFromMessage(entry.Message)

if id == "" {
if entry.Data.RequestGUID != "" {
id = entry.Data.RequestGUID
} else {
id = unspecifiedRequestID
}
}

if existingRequests := ccLog.EntriesForRequest(id); len(existingRequests) > 0 {
entry.Request = existingRequests[0].Request
} else {
// We should only be here if we've never seen the request ID before
entry.Request = &Request{}
entry.Request.RequestID = id
appGUID, _ := appGUIDFromMessage(entry.Message)
if appGUID == "" {
if entry.Data.ProcessGUID != "" {
appGUID = entry.Data.ProcessGUID
} else {
appGUID = unspecifiedAppGUID
}
}

method, uriPath, _ := methodAndURIPathFromMessage(entry.Message)
entry.Request.AppGUID = appGUID
entry.Request.Method = method
entry.Request.URIPath = uriPath

ccLog.requestIDs[appGUID] = append(ccLog.requestIDs[appGUID], id)
}

ccLog.entries[id] = append(ccLog.entries[id], entry)

}

return ccLog, nil
}

func requestIDFromMessage(message string) (string, error) {
rx, err := regexp.Compile(vcapRequestIDRegex)
if err != nil {
return "", fmt.Errorf("Regexp compilation failure: %s", err)
}

var firstMatch string

if firstMatch = rx.FindString(message); firstMatch == "" {
return "", fmt.Errorf("Unable to extract vcap-request-id from %s", message)
}

return strings.Split(firstMatch, " ")[1], nil
}

func appGUIDFromMessage(message string) (string, error) {
rx, err := regexp.Compile(appGUIDRegex)
if err != nil {
return "", fmt.Errorf("App Guid Regexp compilation failure: %s", err)
}

var firstMatch string
if firstMatch = rx.FindString(message); firstMatch == "" {
return "", nil
}

return strings.TrimPrefix(firstMatch, "/v2/apps/"), nil
}

func methodAndURIPathFromMessage(message string) (string, string, error) {
rx, err := regexp.Compile(methodAndURIPathRegex)
if err != nil {
return "", "", fmt.Errorf("Method and URI Path Regexp compilation error: %s", err)
}

var firstMatch string
if firstMatch = rx.FindString(message); firstMatch == "" {
return "", "", nil
}

tokens := strings.Split(firstMatch, " ")

return tokens[1], strings.TrimLeft(tokens[2], `"`), nil
}
Loading

0 comments on commit 1c5cf5b

Please sign in to comment.