Merge pull request #21 from debeando/mysql-sql-parser-slow-queries

feat - mysql sql parser for slow queries.

Author: nicola-strappazzon

mysql/sql/digest/list.go

@@ -0,0 +1,132 @@
+package digest
+
+import (
+	"sort"
+
+	"github.com/debeando/go-common/mysql/sql/parser/slow"
+)
+
+type List []Stats
+
+var FilterByQueryTime float64
+
+func (l List) Index(q slow.Query) (int, bool) {
+	for i := range l {
+		if l[i].ID == q.DigestID {
+			return i, true
+		}
+	}
+	return 0, false
+}
+
+func (l *List) Add(q slow.Query) {
+	index, exists := l.Index(q)
+
+	if !exists {
+		stats := Stats{}
+		stats.Append(q)
+		(*l) = append((*l), stats)
+	} else {
+		(*l)[index].Append(q)
+	}
+}
+
+func (l List) Count() (sum int) {
+	for i := range l {
+		sum = sum + l[i].Count
+	}
+
+	return sum
+}
+
+func (l List) FilterByQueryTime(v float64) {
+	FilterByQueryTime = v
+}
+
+func (l List) Filtered() (sum int) {
+	for i := range l {
+		if l[i].Time.Query < FilterByQueryTime {
+			sum = sum + l[i].Count
+		}
+	}
+
+	return sum
+}
+
+func (l List) Analyzed() (sum int) {
+	for i := range l {
+		if l[i].Time.Query >= FilterByQueryTime {
+			sum = sum + l[i].Count
+		}
+	}
+
+	return sum
+}
+
+func (l List) Unique() (sum int) {
+	for i := range l {
+		if l[i].Time.Query >= FilterByQueryTime {
+			sum++
+		}
+	}
+
+	return sum
+}
+
+func (l List) ScoreMax() (max float64) {
+	for i := range l {
+		if l[i].Time.Query >= FilterByQueryTime {
+			if l[i].Score > max {
+				max = l[i].Score
+			}
+		}
+	}
+
+	return max
+}
+
+func (l List) ScoreMin() (min float64) {
+	for i := range l {
+		if l[i].Time.Query >= FilterByQueryTime {
+			if l[i].Score < min {
+				min = l[i].Score
+			}
+		}
+	}
+
+	return min
+}
+
+func (l *List) Clean() {
+	for i := len((*l)) - 1; i >= 0; i-- {
+		if (*l)[i].Time.Query < FilterByQueryTime {
+			(*l).Remove(i)
+		}
+	}
+}
+
+func (l *List) Remove(i int) {
+	if len((*l)) > i {
+		(*l) = append((*l)[:i], (*l)[i+1:]...)
+	}
+}
+
+// Len is part of sort.Interface.
+func (l List) Len() int {
+	return len(l)
+}
+
+// Less is part of sort.Interface.
+// We use count as the value to sort by
+func (l List) Less(i, j int) bool {
+	return l[i].Score > l[j].Score
+}
+
+// Swap is part of sort.Interface.
+func (l List) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}
+
+func (l List) SortByScore() {
+	sort.Sort(l)
+}

mysql/sql/digest/list_test.go

@@ -0,0 +1,63 @@
+package digest_test
+
+import (
+	"testing"
+
+	"github.com/debeando/go-common/mysql/sql/digest"
+	"github.com/debeando/go-common/mysql/sql/parser/slow"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIndex(t *testing.T) {
+	queries := &digest.List{
+		digest.Stats{ID: "a"},
+		digest.Stats{ID: "b"},
+		digest.Stats{ID: "c"},
+	}
+
+	var cases = []struct {
+		Query slow.Query
+		Index int
+		Exist bool
+	}{
+		{
+			Query: slow.Query{DigestID: "a"},
+			Index: 0,
+			Exist: true,
+		},
+		{
+			Query: slow.Query{DigestID: "b"},
+			Index: 1,
+			Exist: true,
+		},
+		{
+			Query: slow.Query{DigestID: "z"},
+			Index: 0,
+			Exist: false,
+		},
+	}
+
+	for _, test := range cases {
+		i, e := queries.Index(test.Query)
+		assert.Equal(t, i, test.Index)
+		assert.Equal(t, e, test.Exist)
+	}
+}
+
+func TestAddAndCount(t *testing.T) {
+	queries := &digest.List{}
+	queries.Add(slow.Query{DigestID: "a"})
+	queries.Add(slow.Query{DigestID: "a"})
+	queries.Add(slow.Query{DigestID: "b"})
+
+	assert.Equal(t, queries.Count(), 3)
+}
+
+func TestAddAndUnique(t *testing.T) {
+	queries := &digest.List{}
+	queries.Add(slow.Query{DigestID: "a"})
+	queries.Add(slow.Query{DigestID: "a"})
+	queries.Add(slow.Query{DigestID: "b"})
+
+	assert.Equal(t, queries.Unique(), 2)
+}

mysql/sql/digest/stats.go

@@ -0,0 +1,63 @@
+package digest
+
+import (
+	"github.com/debeando/go-common/mysql/sql/parser/slow"
+)
+
+type Stats struct {
+	ID    string
+	Count int
+	Score float64
+	Time  struct {
+		Query float64
+		Lock  float64
+	}
+	Rows struct {
+		Sent     int64
+		Examined int64
+	}
+	Queries []slow.Query
+}
+
+func (s *Stats) Append(q slow.Query) {
+	s.ID = q.DigestID
+	s.Queries = append(s.Queries, q)
+	s.Count = len(s.Queries)
+
+	s.SetQueryTime(q.QueryTime)
+	s.SetLockTime(q.LockTime)
+	s.SetRowsSent(q.RowsSent)
+	s.SetRowsExamined(q.RowsExamined)
+	s.SetScore()
+}
+
+func (s *Stats) SetQueryTime(t float64) {
+	if t > s.Time.Query {
+		s.Time.Query = t
+	}
+}
+
+func (s *Stats) SetLockTime(t float64) {
+	if t > s.Time.Lock {
+		s.Time.Lock = t
+	}
+}
+
+func (s *Stats) SetRowsSent(t int64) {
+	if t > s.Rows.Sent {
+		s.Rows.Sent = t
+	}
+}
+
+func (s *Stats) SetRowsExamined(t int64) {
+	if t > s.Rows.Examined {
+		s.Rows.Examined = t
+	}
+}
+
+func (s *Stats) SetScore() {
+	s.Score = (s.Time.Query * 0.4) +
+		(s.Time.Lock * 0.1) +
+		(float64(s.Count) * 0.2) +
+		(float64(s.Rows.Examined) * 0.3)
+}

mysql/sql/parser/slow/query.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/debeando/go-common/cast"
+	"github.com/debeando/go-common/crypt"
 	"github.com/debeando/go-common/mysql/sql/parser/digest"
 )
 
@@ -83,6 +84,8 @@ func QueryParser(query string) Query {
 	property["query"] = strings.Trim(property["query"], "\n")
 	property["user"] = UserParser(property["user@host"])
 
+	digestQuery := digest.Digest(property["query"])
+
 	return Query{
 		Time:         cast.StringToDateTime(property["time"], "2006-01-02T15:04:05.000000Z"),
 		ID:           cast.StringToInt64(property["id"]),
@@ -93,7 +96,8 @@ func QueryParser(query string) Query {
 		Timestamp:    cast.StringToInt64(property["timestamp"]),
 		Raw:          property["query"],
 		User:         property["user"],
-		Digest:       digest.Digest(property["query"]),
+		Digest:       digestQuery,
+		DigestID:     crypt.MD5(digestQuery),
 	}
 }