Skip to content

Commit 9892487

Browse files
flaviostutzflaviostutz
committed
initial implementation of tagging
1 parent 97fb3bd commit 9892487

File tree

8 files changed

+321
-22
lines changed

8 files changed

+321
-22
lines changed

‎docker-compose.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ services:
55
schelly:
66
build:
77
context: .
8-
target: IMAGE
8+
# target: IMAGE
99
environment:
1010
- LOG_LEVEL=debug
1111
- BACKUP_NAME=test

‎main.go

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
_ "encoding/json"
77
_ "flag"
88
_ "io/ioutil"
9+
_ "math/rand"
910
_ "net/http"
1011
_ "os"
1112
_ "os/signal"

‎schelly/db.go

+171-9
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,23 @@ import (
1313
)
1414

1515
type MaterializedBackup struct {
16-
ID string
17-
StartTime time.Time
18-
EndTime time.Time
19-
tags string
16+
ID string
17+
StartTime time.Time
18+
EndTime time.Time
19+
Tags string
20+
Status string
21+
CustomData string
22+
IsReference int
2023
}
2124

25+
var db = &sql.DB{}
26+
2227
func initDB() error {
23-
db, err := sql.Open("sqlite3", fmt.Sprintf("%s/sqlite.db", options.dataDir))
28+
db0, err := sql.Open("sqlite3", fmt.Sprintf("%s/sqlite.db", options.dataDir))
2429
if err != nil {
2530
return err
2631
}
27-
statement, err1 := db.Prepare("CREATE TABLE IF NOT EXISTS materialized_backup (id TEXT, status TEXT, start_time TIMESTAMP, end_time TIMESTAMP)")
32+
statement, err1 := db0.Prepare("CREATE TABLE IF NOT EXISTS materialized_backup (id TEXT NOT NULL, status TEXT NOT NULL, start_time TIMESTAMP NOT NULL, end_time TIMESTAMP NOT NULL DEFAULT `2000-01-01`, custom_data TEXT NOT NULL DEFAULT ``, tags TEXT NOT NULL DEFAULT ``, is_reference INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`id`))")
2833
if err1 != nil {
2934
return err1
3035
}
@@ -35,6 +40,7 @@ func initDB() error {
3540

3641
os.MkdirAll(options.dataDir, os.ModePerm)
3742

43+
db = db0
3844
logrus.Debug("Database initialized")
3945
return nil
4046
}
@@ -62,9 +68,165 @@ func getCurrentTaskStatus() (string, string, time.Time, error) {
6268
return params[0], params[1], t, nil
6369
}
6470

65-
func createMaterializedBackup(backupID string, tags string, startDate time.Time, endDate time.Time, customData string) (string, error) {
71+
func createMaterializedBackup(backupID string, status string, startDate time.Time, endDate time.Time, customData string) (string, error) {
72+
stmt, err1 := db.Prepare("INSERT INTO materialized_backup (id, status, start_time, end_time, custom_data) values(?,?,?,?,?)")
73+
if err1 != nil {
74+
return "", err1
75+
}
76+
_, err2 := stmt.Exec(backupID, status, startDate, endDate, customData)
77+
if err2 != nil {
78+
return "", err2
79+
}
6680
// rows, _ := db.Query("SELECT id, FROM backup_tasks")
67-
// stmt, err := db.Prepare("INSERT INTO userinfo(username, departname, created) values(?,?,?)")
68-
// res, err := stmt.Exec("astaxie", "研发部门", "2012-12-09")
6981
return backupID, nil
7082
}
83+
84+
func getMaterializedBackup(backupID string) (MaterializedBackup, error) {
85+
rows, err1 := db.Query("SELECT id,tags,status,start_time,end_time,custom_data,is_reference FROM materialized_backup WHERE id='" + backupID + "'")
86+
if err1 != nil {
87+
return MaterializedBackup{}, err1
88+
}
89+
defer rows.Close()
90+
91+
for rows.Next() {
92+
backup := MaterializedBackup{}
93+
err2 := rows.Scan(&backup.ID, &backup.Tags, &backup.Status, &backup.StartTime, &backup.EndTime, &backup.CustomData, &backup.IsReference)
94+
if err2 != nil {
95+
return MaterializedBackup{}, err2
96+
} else {
97+
return backup, nil
98+
}
99+
}
100+
err := rows.Err()
101+
if err != nil {
102+
return MaterializedBackup{}, err
103+
} else {
104+
return MaterializedBackup{}, fmt.Errorf("Backup id %s not found", backupID)
105+
}
106+
}
107+
108+
func getAllMaterializedBackups(limit int) ([]MaterializedBackup, error) {
109+
q := "SELECT id,tags,status,start_time,end_time,custom_data,is_reference FROM materialized_backup ORDER BY start_time DESC")
110+
if limit != 0 {
111+
q = q + fmt.Sprintf(" LIMIT %d", limit)
112+
}
113+
rows, err1 := db.Query(q)
114+
if err1 != nil {
115+
return []MaterializedBackup{}, err1
116+
}
117+
defer rows.Close()
118+
119+
var backups = make([]MaterializedBackup, 0)
120+
for rows.Next() {
121+
backup := MaterializedBackup{}
122+
err2 := rows.Scan(&backup.ID, &backup.Tags, &backup.Status, &backup.StartTime, &backup.EndTime, &backup.CustomData, &backup.IsReference)
123+
if err2 != nil {
124+
return []MaterializedBackup{}, err2
125+
} else {
126+
backups = append(backups, backup)
127+
}
128+
}
129+
err := rows.Err()
130+
if err != nil {
131+
return []MaterializedBackup{}, err
132+
}
133+
return backups, nil
134+
}
135+
136+
// func setTagsMaterializedBackup(backupID string, tags string) error {
137+
// // logrus.Infof("setTagsMaterializedBackup id=%s tags=%s", backupID, tags)
138+
// stmt, err1 := db.Prepare("UPDATE materialized_backup SET tags=? WHERE id=?")
139+
// if err1 != nil {
140+
// return err1
141+
// }
142+
// _, err2 := stmt.Exec(tags, backupID)
143+
// if err2 != nil {
144+
// return err2
145+
// }
146+
147+
// return nil
148+
// }
149+
150+
// func removeTagMaterializedBackup(backupID string, tag string) error {
151+
// backup, err := getMaterializedBackup(backupID)
152+
// if err != nil {
153+
// return err
154+
// } else if backup.ID == "" {
155+
// return fmt.Errorf("Backup %s not found in database", backupID)
156+
// }
157+
158+
// if !tagFound(backup.Tags, tag) {
159+
// return fmt.Errorf("Couldn't find tag '%s' on backup %s. tags=%s", tag, backup.ID, backup.Tags)
160+
// }
161+
162+
// ts := strings.Split(backup.Tags, ",")
163+
// newTags := ""
164+
// for _, t := range ts {
165+
// if t != tag {
166+
// if newTags != "" {
167+
// newTags = newTags + "," + t
168+
// } else {
169+
// newTags = t
170+
// }
171+
// }
172+
// }
173+
// return setTagsMaterializedBackup(backupID, newTags)
174+
// }
175+
176+
// func addTagMaterializedBackup(backupID string, tag string) error {
177+
// // logrus.Infof("addTagMaterializedBackup id=%s tag=%s", backupID, tag)
178+
// backup, err := getMaterializedBackup(backupID)
179+
// if err != nil {
180+
// return err
181+
// } else if backup.ID == "" {
182+
// return fmt.Errorf("Backup %s not found in database", backupID)
183+
// }
184+
185+
// if tagFound(backup.Tags, tag) {
186+
// return fmt.Errorf("Couldn't find tag '%s' on backup %s. tags=%s", tag, backup.ID, backup.Tags)
187+
// }
188+
189+
// newTags := tag
190+
// if backup.Tags != "" {
191+
// newTags = backup.Tags + "," + tag
192+
// }
193+
// return setTagsMaterializedBackup(backupID, newTags)
194+
// }
195+
196+
// func markAsReferenceMaterializedBackup(backupID string, isReference bool) error {
197+
// stmt, err1 := db.Prepare("UPDATE materialized_backup SET is_reference=? WHERE id=?")
198+
// if err1 != nil {
199+
// return err1
200+
// }
201+
// ref := 0
202+
// if isReference {
203+
// ref = 1
204+
// }
205+
// _, err2 := stmt.Exec(ref, backupID)
206+
// if err2 != nil {
207+
// return err2
208+
// }
209+
// return nil
210+
// }
211+
212+
// func isReferenceMaterializedBackup(backupID string) (bool, error) {
213+
// backup, err := getMaterializedBackup(backupID)
214+
// if err != nil {
215+
// return false, err
216+
// } else if backup.ID == "" {
217+
// return false, fmt.Errorf("Backup %s not found in database", backupID)
218+
// } else {
219+
// return backup.IsReference == 1, nil
220+
// }
221+
// }
222+
223+
func tagFound(tags string, tag string) bool {
224+
found := false
225+
ts := strings.Split(","+tags+",", ",")
226+
for _, t := range ts {
227+
if t == tag {
228+
found = true
229+
}
230+
}
231+
return found
232+
}

‎schelly/db_test.go

+96
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,99 @@ func TestStoreTask2(t *testing.T) {
2828
assert.Equal(t, backupStatus, "success", "backupStatus")
2929
assert.Truef(t, backupTime.Sub(time.Now()) < 10000, "backupTime %s", backupTime)
3030
}
31+
32+
func TestGetAllMaterializedBackups(t *testing.T) {
33+
initDB()
34+
_, err0 := createMaterializedBackup("abc", "available", time.Now(), time.Now(), "any")
35+
assert.Nil(t, err0, "err")
36+
_, err0 = createMaterializedBackup("123", "available", time.Now(), time.Now(), "any")
37+
assert.Nil(t, err0, "err")
38+
_, err0 = createMaterializedBackup("xyz", "available", time.Now(), time.Now(), "any")
39+
assert.Nil(t, err0, "err")
40+
backups, err := getAllMaterializedBackups(0)
41+
assert.Nil(t, err, "err")
42+
assert.Equal(t, len(backups), 3, "backups")
43+
}
44+
45+
// func TestMarkAsReferenceMaterializedBackup1(t *testing.T) {
46+
// initDB()
47+
// bid := strconv.Itoa(rand.Int())
48+
// _, err0 := createMaterializedBackup(bid, "available", time.Now(), time.Now(), "any")
49+
// assert.Nil(t, err0, "err")
50+
// err0 = markAsReferenceMaterializedBackup(bid, true)
51+
// assert.Nil(t, err0, "err")
52+
// backup, err1 := getMaterializedBackup(bid)
53+
// assert.Nil(t, err1, "err")
54+
// assert.Equal(t, backup.ID, bid)
55+
// assert.Equal(t, backup.IsReference, 1)
56+
// }
57+
58+
// func TestMarkAsReferenceMaterializedBackup2(t *testing.T) {
59+
// initDB()
60+
// bid := strconv.Itoa(rand.Int())
61+
// _, err0 := createMaterializedBackup(bid, "available", time.Now(), time.Now(), "any")
62+
// assert.Nil(t, err0, "err")
63+
// err0 = markAsReferenceMaterializedBackup(bid, false)
64+
// assert.Nil(t, err0, "err")
65+
// backup, err1 := getMaterializedBackup(bid)
66+
// assert.Nil(t, err1, "err")
67+
// assert.Equal(t, backup.ID, bid)
68+
// assert.Equal(t, backup.IsReference, 0)
69+
// is, err2 := isReferenceMaterializedBackup(bid)
70+
// assert.Nil(t, err2, "err")
71+
// assert.False(t, is, "false")
72+
// err0 = markAsReferenceMaterializedBackup(bid, true)
73+
// assert.Nil(t, err0, "err")
74+
// is, err2 = isReferenceMaterializedBackup(bid)
75+
// assert.Nil(t, err2, "true")
76+
// assert.True(t, is, "false")
77+
// }
78+
79+
// func TestAddRemoveTagMaterializedBackup1(t *testing.T) {
80+
// initDB()
81+
// bid := strconv.Itoa(rand.Int())
82+
// _, err0 := createMaterializedBackup(bid, "available", time.Now(), time.Now(), "any")
83+
84+
// assert.Nil(t, err0, "err")
85+
// err0 = addTagMaterializedBackup(bid, "monthly")
86+
// assert.Nil(t, err0, "err")
87+
88+
// err0 = removeTagMaterializedBackup(bid, "monthly")
89+
// assert.Nil(t, err0, "err")
90+
91+
// err0 = addTagMaterializedBackup(bid, "minutely")
92+
// assert.Nil(t, err0, "err")
93+
// err0 = addTagMaterializedBackup(bid, "monthly")
94+
// assert.Nil(t, err0, "err")
95+
// backup, err1 := getMaterializedBackup(bid)
96+
// assert.Nil(t, err1, "err")
97+
// assert.True(t, strings.Contains(backup.Tags, "minutely"), "contains tags")
98+
// assert.True(t, strings.Contains(backup.Tags, "monthly"), "contains tags")
99+
// assert.False(t, strings.Contains(backup.Tags, "yearly"), "contains tags")
100+
101+
// err0 = addTagMaterializedBackup(bid, "yearly")
102+
// assert.Nil(t, err0, "err")
103+
// backup, err1 = getMaterializedBackup(bid)
104+
// assert.Nil(t, err1, "err")
105+
// assert.True(t, strings.Contains(backup.Tags, "yearly"), "contains tags")
106+
107+
// err0 = removeTagMaterializedBackup(bid, "minutely")
108+
// assert.Nil(t, err0, "err")
109+
// err0 = removeTagMaterializedBackup(bid, "monthly")
110+
// assert.Nil(t, err0, "err")
111+
// err0 = removeTagMaterializedBackup(bid, "monthly")
112+
// assert.NotNil(t, err0, "err")
113+
// err0 = removeTagMaterializedBackup(bid, "yearly")
114+
// assert.Nil(t, err0, "err")
115+
// backup, err1 = getMaterializedBackup(bid)
116+
// assert.Nil(t, err1, "err")
117+
// assert.False(t, strings.Contains(backup.Tags, "yearly"), "contains tags")
118+
// assert.Equal(t, backup.Tags, "", "empty tags")
119+
120+
// err1 = removeTagMaterializedBackup(bid, "monthly")
121+
// assert.NotNil(t, err1, "err")
122+
// backup, err1 = getMaterializedBackup(bid)
123+
// assert.Nil(t, err1, "err")
124+
// assert.Equal(t, backup.ID, bid)
125+
// assert.Equal(t, backup.IsReference, 0)
126+
// }

‎schelly/task-backup.go

+34-4
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,8 @@ func runBackupTask() {
1010
start := time.Now()
1111
logrus.Info("")
1212
logrus.Info(">>>> BACKUP TASK")
13-
logrus.Info("")
1413

15-
logrus.Debug("Checking if there is another running backup")
14+
logrus.Debug("Checking if there is another backup running")
1615

1716
backupID, backupStatus, backupDate, err := getCurrentTaskStatus()
1817
if err != nil {
@@ -33,12 +32,13 @@ func runBackupTask() {
3332
} else {
3433
if resp.Status == "available" {
3534
logrus.Infof("Backup executed successfuly and is already available. id=%s; status=%s message=%s", resp.ID, resp.Status, resp.Message)
36-
mid, err1 := createMaterializedBackup(resp.ID, "", startPostTime, time.Now(), "")
35+
mid, err1 := createMaterializedBackup(resp.ID, resp.Status, startPostTime, time.Now(), resp.Message)
3736
if err1 != nil {
3837
logrus.Errorf("Couldn't create materialized backup on database. err=%s", err1)
3938
} else {
4039
logrus.Infof("Materialized backup reference saved to database successfuly. id=%s", mid)
4140
setCurrentTaskStatus(resp.ID, resp.Status, startPostTime)
41+
tagLastBackups()
4242
}
4343
} else if resp.Status == "running" {
4444
logrus.Infof("Backup invoked successfuly but it is still running in background (not available yet). Starting to check for completion from time to time. id=%s; status=%s message=%s", resp.ID, resp.Status, resp.Message)
@@ -52,6 +52,36 @@ func runBackupTask() {
5252
logrus.Infof("Backup task done. elapsed=%s", elapsed)
5353
}
5454

55+
func tagLastBackups() {
56+
logrus.Infof("Tagging backups")
57+
58+
backups, err := getAllMaterializedBackups(2)
59+
if len(backups) != 2 {
60+
logrus.Infof("Too few backups yet n=%s. Skipping", len(backups))
61+
return
62+
}
63+
64+
b1 := backups[1]
65+
b0 := backups[0]
66+
67+
setTagsMaterializedBackup(b0.ID, "secondly,minutely,hourly,weekly,daily,monthly,yearly")
68+
69+
//secondly
70+
71+
//minutely
72+
73+
//hourly
74+
75+
//weekly
76+
77+
//daily
78+
79+
//monthly
80+
81+
//yearly
82+
83+
}
84+
5585
func checkBackupTask() {
5686
logrus.Debug("checkBackupTask")
5787
backupID, backupStatus, backupDate, err := getCurrentTaskStatus()
@@ -66,7 +96,7 @@ func checkBackupTask() {
6696
} else {
6797
if resp.Status != backupStatus {
6898
logrus.Infof("Backup %s finished on backend server. status=%s", backupID, resp.Status)
69-
mid, err1 := createMaterializedBackup(resp.ID, "", backupDate, time.Now(), "")
99+
mid, err1 := createMaterializedBackup(resp.ID, resp.Status, backupDate, time.Now(), resp.Message)
70100
if err1 != nil {
71101
logrus.Errorf("Couldn't create materialized backup on database. err=%s", err1)
72102
} else {

0 commit comments

Comments
 (0)