Skip to content

Commit 71ffacd

Browse files
DutchJavaDevDutchJavaDev
committed
task logging update
1 parent 7bcc521 commit 71ffacd

File tree

11 files changed

+213
-122
lines changed

11 files changed

+213
-122
lines changed

‎MyMusicBoxApi/database/db.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ const ReturningIdParameter = "RETURNING"
2020
const ReturningIdParameterLower = "returning"
2121
const DatabaseDriver = "postgres"
2222
const MigrationFolder = "migration_scripts"
23-
const MaxOpenConnections = 10
24-
const MaxIdleConnections = 5
23+
const MaxOpenConnections = 15
24+
const MaxIdleConnections = 10
2525
const MaxConnectionIdleTimeInMinutes = 10
2626
const MaxConnectionLifeTimeInMinutes = 10
2727

@@ -164,7 +164,7 @@ func (base *BaseTable) QueryRows(query string) (*sql.Rows, error) {
164164
}
165165

166166
func ApplyMigrations() {
167-
logging.Info("Applying migrations...")
167+
logging.Info("Checking for database migration files")
168168
// files will be sorted by filename
169169
// to make sure the migrations are executed in order
170170
// this naming convention must be used

‎MyMusicBoxApi/database/migrationtable.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,22 @@ func (table *MigrationTable) Insert(filename string, contents string) (err error
3030
}
3131

3232
func (table *MigrationTable) ApplyMigration(query string) (err error) {
33-
return table.NonScalarQuery(query)
33+
34+
transaction, err := table.DB.Begin()
35+
36+
if err != nil {
37+
logging.Error(fmt.Sprintf("Failed to creare transaction %s", err.Error()))
38+
return
39+
}
40+
41+
_, err = transaction.Exec(query)
42+
43+
if err != nil {
44+
logging.Error(fmt.Sprintf("Failed to execute query %s", err.Error()))
45+
return
46+
}
47+
48+
return transaction.Commit()
3449
}
3550

3651
func (table *MigrationTable) GetCurrentAppliedMigrationFileName() (fileName string, err error) {

‎MyMusicBoxApi/database/tasklogtable.go‎

Lines changed: 47 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,16 @@ package database
22

33
import (
44
"context"
5-
"fmt"
6-
"musicboxapi/logging"
75
"musicboxapi/models"
8-
"time"
96
)
107

118
type ITasklogTable interface {
12-
InsertTaskLog() (lastInsertedId int, err error)
13-
UpdateTaskLogStatus(taskId int, nStatus int) (err error)
14-
EndTaskLog(taskId int, nStatus int, data []byte) (err error)
15-
UpdateTaskLogError(params ...any) (err error)
16-
GetTaskLogs(ctx context.Context) ([]models.TaskLog, error)
9+
GetParentChildLogs(ctx context.Context) ([]models.ParentTaskLog, error)
10+
CreateParentTaskLog(url string) (models.ParentTaskLog, error)
11+
CreateChildTaskLog(parent models.ParentTaskLog) (models.ChildTaskLog, error)
12+
UpdateChildTaskLogStatus(child models.ChildTaskLog) error
13+
ChildTaskLogDone(child models.ChildTaskLog) error
14+
ChildTaskLogError(child models.ChildTaskLog) error
1715
}
1816

1917
type TasklogTable struct {
@@ -25,59 +23,57 @@ func NewTasklogTableInstance() *TasklogTable {
2523
BaseTable: NewBaseTableInstance(),
2624
}
2725
}
28-
29-
func (table *TasklogTable) InsertTaskLog() (lastInsertedId int, error error) {
30-
query := `INSERT INTO TaskLog (Status) VALUES($1) RETURNING Id`
31-
32-
lastInsertedId, err := table.InsertWithReturningId(query, int(models.Pending))
33-
34-
return lastInsertedId, err
26+
func (table *TasklogTable) GetParentChildLogs(ctx context.Context) ([]models.ParentTaskLog, error) {
27+
return make([]models.ParentTaskLog, 0), nil
3528
}
29+
func (table *TasklogTable) CreateParentTaskLog(url string) (models.ParentTaskLog, error) {
30+
query := "INSERT INTO ParentTaskLog (Url) Values($1) RETURNING Id"
3631

37-
func (table *TasklogTable) UpdateTaskLogStatus(taskId int, nStatus int) (error error) {
38-
query := `UPDATE TaskLog SET Status = $1 WHERE Id = $2`
39-
40-
return table.NonScalarQuery(query, nStatus, taskId)
41-
}
32+
id, err := table.InsertWithReturningId(query, url)
4233

43-
func (table *TasklogTable) EndTaskLog(taskId int, nStatus int, data []byte) error {
44-
query := `UPDATE TaskLog SET Status = $1, OutputLog = $2, EndTime = $3 WHERE Id = $4`
45-
46-
return table.NonScalarQuery(query, nStatus, data, time.Now(), taskId)
47-
}
34+
if err != nil {
35+
return models.ParentTaskLog{}, err
36+
}
4837

49-
func (table *TasklogTable) UpdateTaskLogError(params ...any) error {
50-
query := `UPDATE TaskLog
51-
SET Status = $1, OutputLog = $2, EndTime = $3
52-
WHERE Id = $4`
53-
return table.NonScalarQuery(query, params...)
38+
return models.ParentTaskLog{
39+
Id: id,
40+
Url: url,
41+
}, nil
5442
}
43+
func (table *TasklogTable) CreateChildTaskLog(parent models.ParentTaskLog) (models.ChildTaskLog, error) {
44+
query := "INSERT INTO ChildTaskLog (ParentId, Status) VALUES($1,$2) RETURNING Id"
5545

56-
func (table *TasklogTable) GetTaskLogs(ctx context.Context) ([]models.TaskLog, error) {
57-
query := `SELECT Id, StartTime, EndTime, Status, OutputLog FROM TaskLog ORDER BY Id desc` // get the latest first
46+
defaultStatus := int(models.Pending)
5847

59-
rows, err := table.QueryRowsContex(ctx, query)
48+
id, err := table.InsertWithReturningId(query, parent.Id, defaultStatus)
6049

6150
if err != nil {
62-
logging.Error(fmt.Sprintf("QueryRow error: %s", err.Error()))
63-
return nil, err
51+
return models.ChildTaskLog{}, err
6452
}
65-
defer rows.Close()
6653

67-
var tasklog models.TaskLog
68-
69-
tasks := make([]models.TaskLog, 0)
70-
71-
for rows.Next() {
72-
scanError := rows.Scan(&tasklog.Id, &tasklog.StartTime, &tasklog.EndTime, &tasklog.Status, &tasklog.OutputLog)
73-
74-
if scanError != nil {
75-
logging.Error(fmt.Sprintf("Scan error: %s", scanError.Error()))
76-
continue
77-
}
78-
79-
tasks = append(tasks, tasklog)
54+
return models.ChildTaskLog{
55+
Id: id,
56+
ParentId: parent.Id,
57+
Status: defaultStatus,
58+
}, nil
59+
}
60+
func (table *TasklogTable) UpdateChildTaskLogStatus(child models.ChildTaskLog) error {
61+
62+
if child.Status == int(models.Downloading) {
63+
// set the start time to now
64+
query := "UPDATE ChildTaskLog SET StartTime = CURRENT_TIMESTAMP, Status = $1 WHERE Id = $2"
65+
return table.NonScalarQuery(query, child.Status, child.Id)
66+
} else {
67+
// just update
68+
query := "UPDATE ChildTaskLog SET Status = $1 WHERE Id = $2"
69+
return table.NonScalarQuery(query, child.Status, child.Id)
8070
}
81-
82-
return tasks, nil
71+
}
72+
func (table *TasklogTable) ChildTaskLogDone(child models.ChildTaskLog) error {
73+
query := "UPDATE ChildTaskLog SET Status = $1, OutputLog = $2, EndTime = CURRENT_TIMESTAMP WHERE Id = $3"
74+
return table.NonScalarQuery(query, int(models.Done), child.OutputLog, child.Id)
75+
}
76+
func (table *TasklogTable) ChildTaskLogError(child models.ChildTaskLog) error {
77+
query := "UPDATE ChildTaskLog SET Status = $1, OutputLog = $2, EndTime = CURRENT_TIMESTAMP WHERE Id = $3"
78+
return table.NonScalarQuery(query, int(models.Error), child.OutputLog, child.Id)
8379
}

‎MyMusicBoxApi/http/download.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package http
22

33
import (
4-
"musicboxapi/database"
54
"musicboxapi/models"
65
"musicboxapi/service"
76
"net/http"
@@ -25,15 +24,15 @@ func DownloadRequest(ctx *gin.Context) {
2524
return
2625
}
2726

28-
tasklogTable := database.NewTasklogTableInstance()
27+
//tasklogTable := database.NewTasklogTableInstance()
2928
// Insert a new task
30-
taskId, err := tasklogTable.InsertTaskLog()
29+
// parentTask, err := tasklogTable.CreateParentTaskLog(request.Url)
3130

3231
if err != nil {
3332
ctx.JSON(http.StatusInternalServerError, models.ErrorResponse(err))
3433
return
3534
}
3635

37-
go service.StartDownloadTask(taskId, request)
38-
ctx.JSON(http.StatusOK, models.OkResponse(gin.H{"taskId": taskId}, "Started task"))
36+
go service.StartDownloadTask(request)
37+
ctx.JSON(http.StatusOK, models.OkResponse(gin.H{"": ""}, "Created"))
3938
}

‎MyMusicBoxApi/http/tasklog.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ type TaskLogHandler struct {
1515

1616
func (handler *TaskLogHandler) FetchTaskLogs(ctx *gin.Context) {
1717

18-
logs, err := handler.TasklogTable.GetTaskLogs(ctx.Request.Context())
18+
logs, err := handler.TasklogTable.GetParentChildLogs(ctx.Request.Context())
1919

2020
if err != nil {
2121
ctx.JSON(http.StatusInternalServerError, models.ErrorResponse(err.Error()))

‎MyMusicBoxApi/migration_scripts/1 tasklog update.sql‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
-- Used for ytdlp output
2+
CREATE TABLE IF NOT EXISTS ParentTaskLog(
3+
Id SERIAL PRIMARY KEY,
4+
Url VARCHAR(72) NOT NULL,
5+
AddTime TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
6+
);
7+
8+
CREATE TABLE IF NOT EXISTS ChildTaskLog(
9+
Id SERIAL PRIMARY KEY,
10+
ParentId INTEGER NOT NULL REFERENCES ParentTaskLog(Id) ON DELETE CASCADE,
11+
StartTime TIMESTAMP WITH TIME ZONE NULL,
12+
EndTime TIMESTAMP WITH TIME ZONE NULL,
13+
Status INTEGER NOT NULL CHECK (Status BETWEEN 0 AND 4),
14+
OutputLog JSONB
15+
);
16+
17+
COMMENT ON COLUMN ChildTaskLog.Status IS '0=Pending, 1=Downloading, 2=Updating, 3=Done, 4=Error';
18+
19+
CREATE INDEX IF NOT EXISTS idx_parenttasklog_url ON ParentTaskLog(Url);
20+
CREATE INDEX IF NOT EXISTS idx_parenttasklog_addtime ON ParentTaskLog(AddTime);
21+
22+
CREATE INDEX IF NOT EXISTS idx_childtasklog_starttime ON ChildTaskLog(StartTime);
23+
CREATE INDEX IF NOT EXISTS idx_childtasklog_endtime ON ChildTaskLog(EndTime);
24+
CREATE INDEX IF NOT EXISTS idx_childtasklog_status ON ChildTaskLog(Status);
25+
26+
--- Remove prev indexs
27+
DROP INDEX IF EXISTS idx_tasklog_starttime;
28+
DROP INDEX IF EXISTS idx_tasklog_status;
29+
30+
DROP TABLE IF EXISTS TaskLog;

‎MyMusicBoxApi/models/database.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ const (
1515
Error
1616
)
1717

18+
// TODO Delete
1819
type TaskLog struct {
1920
Id int `json:"id" db:"id"`
2021
StartTime time.Time `json:"startTime" db:"starttime"`
@@ -56,3 +57,18 @@ type MigrationFile struct {
5657
Name string `json:"filename" db:"filename"`
5758
AppliedOn time.Time `json:"appliedon" db:"appliedon"`
5859
}
60+
61+
type ParentTaskLog struct {
62+
Id int `db:"id" json:"id"`
63+
Url string `db:"url" json:"url"`
64+
AddTime time.Time `db:"add_time" json:"add_time"`
65+
}
66+
67+
type ChildTaskLog struct {
68+
Id int `db:"id" json:"id"`
69+
ParentId int `db:"parent_id" json:"parent_id"`
70+
StartTime time.Time `db:"start_time" json:"start_time"`
71+
EndTime *time.Time `db:"end_time" json:"end_time,omitempty"`
72+
Status int `db:"status" json:"status"`
73+
OutputLog json.RawMessage `db:"output_log" json:"output_log"`
74+
}

‎MyMusicBoxApi/reset‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@ sudo docker compose down
99
sudo rm -r data/
1010

1111
# restart dev docker
12-
sudo docker compose up -d
12+
# sudo docker compose up -d

‎MyMusicBoxApi/run‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ for folder in "${FOLDERS[@]}"; do
1313

1414
if [[ "music_dev/images" == $folder ]]; then
1515
echo "Copied default image"
16-
cp default/images/* "$IMAGES_FOLDER"
16+
cp default/* "$IMAGES_FOLDER"
1717
fi
1818
else
1919
echo "Exists: $folder"

‎MyMusicBoxApi/service/playlist.go‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
)
1717

1818
func downloadPlaylist(
19-
taskId int,
19+
parentTask models.ParentTaskLog,
2020
storage string,
2121
archiveFileName string,
2222
idsFileName string,
@@ -72,6 +72,15 @@ func downloadPlaylist(
7272
})
7373

7474
if err != nil {
75+
76+
errorChildLog, err := tasklogTable.CreateChildTaskLog(parentTask)
77+
78+
json, err := json.Marshal(err)
79+
80+
errorChildLog.OutputLog = json
81+
82+
tasklogTable.ChildTaskLogError(errorChildLog)
83+
7584
logging.Error(fmt.Sprintf("[Creating custom playlist error]: %s", err.Error()))
7685
return
7786
}
@@ -86,9 +95,6 @@ func downloadPlaylist(
8695
_ = os.Rename(oldpath, newpath)
8796
}
8897

89-
// Update task status
90-
tasklogTable.UpdateTaskLogStatus(taskId, int(models.Downloading))
91-
9298
defaultSettings := ytdlp.New().
9399
ExtractAudio().
94100
AudioQuality("0").
@@ -105,27 +111,34 @@ func downloadPlaylist(
105111
Output(storage + "/%(id)s.%(ext)s").
106112
Cookies("selenium/cookies_netscape")
107113

108-
var outputLogs map[string]string
109-
110-
outputLogs = make(map[string]string)
111-
112-
hasError := false
113-
114114
for id := range downloadCount {
115115
name := names[id]
116116
if canDownload(name) {
117+
118+
childTask, _ := tasklogTable.CreateChildTaskLog(parentTask)
119+
120+
childTask.Status = int(models.Downloading)
121+
122+
tasklogTable.UpdateChildTaskLogStatus(childTask)
123+
117124
ytdlpInstance := defaultSettings.Clone()
118125

119126
result, err := ytdlpInstance.Run(context.Background(), fmt.Sprintf("https://www.youtube.com/watch?v=%s", ids[id]))
120127

121-
outputLogs[ids[id]] = result.Stdout
128+
// outputLogs[ids[id]] = result.Stdout
122129

123130
if err != nil {
124-
hasError = true
131+
json, _ := json.Marshal(result.Stdout)
132+
childTask.OutputLog = json
133+
tasklogTable.ChildTaskLogError(childTask)
125134
logging.Error(fmt.Sprintf("Failed to download %s, error:%s", ids[id], err.Error()))
126135
continue
127136
}
128137

138+
childTask.Status = int(models.Updating)
139+
140+
tasklogTable.UpdateChildTaskLogStatus(childTask)
141+
129142
var song models.Song
130143

131144
song.Name = names[id]
@@ -142,20 +155,13 @@ func downloadPlaylist(
142155
newpath := fmt.Sprintf("%s/%s", imagesFolder, song.ThumbnailPath)
143156

144157
_ = os.Rename(oldpath, newpath)
145-
}
146-
}
147158

148-
json, err := json.Marshal(outputLogs)
159+
json, _ := json.Marshal(result.OutputLogs)
149160

150-
status := models.Done
161+
childTask.OutputLog = json
151162

152-
if hasError {
153-
status = models.Error
154-
}
155-
156-
err = tasklogTable.EndTaskLog(taskId, int(status), json)
157-
if err != nil {
158-
logging.Error(fmt.Sprintf("Failed to update tasklog: %s", err.Error()))
163+
tasklogTable.ChildTaskLogDone(childTask)
164+
}
159165
}
160166
}
161167

0 commit comments

Comments
 (0)