Feature/script migrations (#44)

* base for script migrations

* not going to modify this anymore

* mod update

* update database on the fly

* base tests

* this should have been here fromt the start

* cleaner to read

* fingers crossed

* tweaks

* run go test

* name lol

* wooooops

* versionnnnnnn

* split github actions

* jesus

* third time is the charm

* bruhhhh

* yea boi

* yea boii 2

* why was this added

* welcome back sum

* meh

Author: DutchJavaDev

.github/workflows/backend.yml

@@ -0,0 +1,33 @@
+name: Api Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+    types:
+      - opened
+      - synchronize
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  api_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.24.3'
+          cache-dependency-path: MyMusicBoxApi/go.sum
+
+      - name: Tidy modules (sync go.mod & go.sum)
+        working-directory: MyMusicBoxApi
+        run: go mod tidy
+
+      - name: Test
+        working-directory: MyMusicBoxApi
+        run: go test -v ./...

.github/workflows/main.yml .github/workflows/frontend.yml.github/workflows/main.yml renamed to .github/workflows/frontend.yml

@@ -13,8 +13,6 @@
 /bin/
 /pkg/
 
-# Go modules
-go.sum
 
 # IDE/editor files
 .vscode/

.gitignore

@@ -0,0 +1,29 @@
+package configuration
+
+import (
+	"flag"
+	"musicboxapi/models"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadConfiguration(t *testing.T) {
+	// Arrange
+	// Reset flags
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	os.Args = []string{"cmd", "-port=8081", "-devurl", "-sourceFolder=dev_music", "-outputExtension=opus"}
+
+	// Act
+	LoadConfiguration()
+
+	// Assert
+	var config models.Config = Config
+
+	assert.Equal(t, "8081", config.DevPort)
+	assert.Equal(t, "opus", config.OutputExtension)
+	assert.Equal(t, "dev_music", config.SourceFolder)
+	assert.Equal(t, true, config.UseDevUrl)
+}

MyMusicBoxApi/configuration/util_test.go

@@ -7,12 +7,23 @@ import (
 	"musicboxapi/configuration"
 	"musicboxapi/logging"
 	"os"
+	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
 	_ "github.com/lib/pq"
 )
 
+const ReturningIdParameter = "RETURNING"
+const ReturningIdParameterLower = "returning"
+const DatabaseDriver = "postgres"
+const MigrationFolder = "migration_scripts"
+const MaxOpenConnections = 10
+const MaxIdleConnections = 5
+const MaxConnectionIdleTimeInMinutes = 10
+const MaxConnectionLifeTimeInMinutes = 10
+
 var DbInstance *sql.DB
 
 type BaseTable struct {
@@ -29,10 +40,12 @@ func CreateDatabasConnectionPool() error {
 
 	// Will throw an error if its missing a method implementation from interface
 	// will throw a compile time error
+	// Should create test for these?
 	var _ ISongTable = (*SongTable)(nil)
 	var _ IPlaylistTable = (*PlaylistTable)(nil)
 	var _ IPlaylistsongTable = (*PlaylistsongTable)(nil)
 	var _ ITasklogTable = (*TasklogTable)(nil)
+	var _ IMigrationTable = (*MigrationTable)(nil)
 
 	baseConnectionString := "user=postgres dbname=postgres password=%s %s sslmode=disable"
 	password := os.Getenv("POSTGRES_PASSWORD")
@@ -44,17 +57,18 @@ func CreateDatabasConnectionPool() error {
 
 	connectionString := fmt.Sprintf(baseConnectionString, password, host)
 
-	DB, err := sql.Open("postgres", connectionString)
+	DB, err := sql.Open(DatabaseDriver, connectionString)
 
 	if err != nil {
 		logging.Error(fmt.Sprintf("Failed to init database connection: %s", err.Error()))
+		logging.ErrorStackTrace(err)
 		return err
 	}
 
-	DB.SetMaxOpenConns(10)
-	DB.SetMaxIdleConns(5)
-	DB.SetConnMaxIdleTime(1 * time.Minute)
-	DB.SetConnMaxLifetime(5 * time.Minute)
+	DB.SetMaxOpenConns(MaxOpenConnections)
+	DB.SetMaxIdleConns(MaxIdleConnections)
+	DB.SetConnMaxIdleTime(MaxConnectionIdleTimeInMinutes * time.Minute)
+	DB.SetConnMaxLifetime(MaxConnectionLifeTimeInMinutes * time.Minute)
 
 	DbInstance = DB
 
@@ -64,8 +78,7 @@ func CreateDatabasConnectionPool() error {
 // Base methods
 func (base *BaseTable) InsertWithReturningId(query string, params ...any) (lastInsertedId int, err error) {
 
-	if !strings.Contains(query, "RETURNING") {
-		logging.Error("Query does not contain RETURNING keyword")
+	if !strings.Contains(query, ReturningIdParameter) {
 		return -1, errors.New("Query does not contain RETURNING keyword")
 	}
 
@@ -74,43 +87,45 @@ func (base *BaseTable) InsertWithReturningId(query string, params ...any) (lastI
 	statement, err := transaction.Prepare(query)
 
 	if err != nil {
-		transaction.Rollback()
-		logging.Error(fmt.Sprintf("Prepared statement error: %s", err.Error()))
+		logging.ErrorStackTrace(err)
 		return -1, err
 	}
 	defer statement.Close()
 
 	err = statement.QueryRow(params...).Scan(&lastInsertedId)
 
 	if err != nil {
-		logging.Error(fmt.Sprintf("Queryrow error: %s", err.Error()))
+		logging.ErrorStackTrace(err)
 		transaction.Rollback()
 		return -1, err
 	}
 
 	err = transaction.Commit()
 
 	if err != nil {
-		logging.Error(fmt.Sprintf("Transaction commit error: %s", err.Error()))
+		logging.ErrorStackTrace(err)
 		transaction.Rollback()
 		return -1, err
 	}
 
 	return lastInsertedId, nil
 }
+
 func (base *BaseTable) NonScalarQuery(query string, params ...any) (error error) {
 
 	transaction, err := base.DB.Begin()
 
 	if err != nil {
 		logging.Error(fmt.Sprintf("Transaction error: %s", err.Error()))
+		logging.ErrorStackTrace(err)
 		return err
 	}
 
 	statement, err := transaction.Prepare(query)
 
 	if err != nil {
 		logging.Error(fmt.Sprintf("Prepared statement error: %s", err.Error()))
+		logging.ErrorStackTrace(err)
 		return err
 	}
 
@@ -119,20 +134,99 @@ func (base *BaseTable) NonScalarQuery(query string, params ...any) (error error)
 	_, err = statement.Exec(params...)
 
 	if err != nil {
-		logging.Error(fmt.Sprintf("Exec error: %s", err.Error()))
-		logging.Error(fmt.Sprintf("Query: %s", query))
-		for index := range params {
-			logging.Error(params[index])
-		}
+		logging.ErrorStackTrace(err)
 		return err
 	}
 
 	err = transaction.Commit()
 
 	if err != nil {
 		logging.Error(fmt.Sprintf("Transaction commit error: %s", err.Error()))
+		logging.ErrorStackTrace(err)
 		return err
 	}
 
 	return nil
 }
+
+func ApplyMigrations() {
+	logging.Info("Applying migrations...")
+	// files will be sorted by filename
+	// to make sure the migrations are executed in order
+	// this naming convention must be used
+	// 0 initial script.sql
+	// 1 update column.sql
+	// etc....
+	// entries are sorted by file name
+	dirs, err := os.ReadDir(MigrationFolder)
+
+	if err != nil {
+		logging.ErrorStackTrace(err)
+		return
+	}
+
+	migrationTable := NewMigrationTableInstance()
+
+	currentMigrationFileName, err := migrationTable.GetCurrentAppliedMigrationFileName()
+
+	// start at -1 if lastMigrationFileName is empty OR migration table does not exists
+	// start applying from 0
+	if currentMigrationFileName == "" || err != nil {
+		if strings.Contains(err.Error(), `relation "migration" does not exist`) {
+			logging.Info("First time running database script migrations")
+		} else {
+			logging.ErrorStackTrace(err)
+			return
+		}
+
+		// makes sure we start at script 0
+		currentMigrationFileName = "-1 nil.sql"
+	}
+
+	currentMigrationFileId, err := strconv.Atoi(strings.Split(currentMigrationFileName, " ")[0])
+
+	if err != nil {
+		logging.ErrorStackTrace(err)
+		return
+	}
+
+	for _, migrationFile := range dirs {
+		filePath := filepath.Join(MigrationFolder, migrationFile.Name())
+
+		migrationFileId, err := strconv.Atoi(strings.Split(migrationFile.Name(), " ")[0])
+
+		if err != nil {
+			logging.ErrorStackTrace(err)
+			continue
+		}
+
+		if migrationFileId <= currentMigrationFileId {
+			continue
+		}
+
+		migrationFileContents, err := os.ReadFile(filePath)
+
+		if err != nil {
+			logging.ErrorStackTrace(err)
+			continue
+		}
+
+		err = migrationTable.ApplyMigration(string(migrationFileContents))
+
+		if err != nil {
+			logging.Error(fmt.Sprintf("Failed to apply %s", migrationFile.Name()))
+			logging.ErrorStackTrace(err)
+		} else {
+			err = migrationTable.Insert(migrationFile.Name(), string(migrationFileContents))
+
+			if err != nil {
+				logging.Error(fmt.Sprintf("Failed to insert migration entry %s: %s", migrationFile.Name(), err.Error()))
+				logging.ErrorStackTrace(err)
+				return
+			}
+
+			logging.Info(fmt.Sprintf("Applied script: %s", migrationFile.Name()))
+		}
+
+	}
+}