Skip to content

Commit f8a868e

Browse files
PriveeteePriveetee
authored
Merge pull request #9 from Priveetee/dev
fix: canonicalize subscription urls and deduplicate records
2 parents 7b3ce19 + 79bf515 commit f8a868e

8 files changed

Lines changed: 248 additions & 7 deletions

‎src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ object DatabaseFactory {
110110
exec("CREATE UNIQUE INDEX IF NOT EXISTS users_public_username_unique ON users (public_username)")
111111
DatabasePrimaryKeyMigrations.apply()
112112
DatabaseIndexMigrations.apply()
113+
DatabaseSubscriptionsCanonicalMigration.apply()
113114
}
114115
}
115116

‎src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package dev.typetype.server.db
2+
3+
import dev.typetype.server.db.tables.SubscriptionsTable
4+
import dev.typetype.server.services.ChannelUrlCanonicalizer
5+
import org.jetbrains.exposed.v1.jdbc.deleteAll
6+
import org.jetbrains.exposed.v1.jdbc.insert
7+
import org.jetbrains.exposed.v1.jdbc.selectAll
8+
9+
object DatabaseSubscriptionsCanonicalMigration {
10+
fun apply() {
11+
val rows = SubscriptionsTable.selectAll().map {
12+
SubscriptionRow(
13+
userId = it[SubscriptionsTable.userId],
14+
channelUrl = it[SubscriptionsTable.channelUrl],
15+
name = it[SubscriptionsTable.name],
16+
avatarUrl = it[SubscriptionsTable.avatarUrl],
17+
subscribedAt = it[SubscriptionsTable.subscribedAt],
18+
)
19+
}
20+
if (rows.isEmpty()) return
21+
val normalized = rows.map { row ->
22+
row.copy(channelUrl = ChannelUrlCanonicalizer.canonicalize(row.channelUrl))
23+
}
24+
val deduped = normalized
25+
.groupBy { it.userId to it.channelUrl }
26+
.values
27+
.map { group ->
28+
group.maxWith(compareBy<SubscriptionRow> { it.subscribedAt }
29+
.thenBy { it.name }
30+
.thenBy { it.avatarUrl })
31+
}
32+
val needsCanonicalization = rows.zip(normalized).any { (before, after) -> before.channelUrl != after.channelUrl }
33+
val needsDeduplication = deduped.size != normalized.size
34+
if (!needsCanonicalization && !needsDeduplication) return
35+
SubscriptionsTable.deleteAll()
36+
deduped.forEach { row ->
37+
SubscriptionsTable.insert {
38+
it[SubscriptionsTable.userId] = row.userId
39+
it[SubscriptionsTable.channelUrl] = row.channelUrl
40+
it[SubscriptionsTable.name] = row.name
41+
it[SubscriptionsTable.avatarUrl] = row.avatarUrl
42+
it[SubscriptionsTable.subscribedAt] = row.subscribedAt
43+
}
44+
}
45+
}
46+
47+
private data class SubscriptionRow(
48+
val userId: String,
49+
val channelUrl: String,
50+
val name: String,
51+
val avatarUrl: String,
52+
val subscribedAt: Long,
53+
)
54+
}

‎src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package dev.typetype.server.services
2+
3+
import java.net.URI
4+
5+
object ChannelUrlCanonicalizer {
6+
fun canonicalize(raw: String): String {
7+
val trimmed = raw.trim()
8+
if (trimmed.isEmpty()) return trimmed
9+
val uri = runCatching { URI(trimmed) }.getOrNull() ?: return trimmed
10+
val host = uri.host?.lowercase()?.takeIf { it.isNotBlank() } ?: return trimmed
11+
val path = normalizePath(uri.path)
12+
val port = normalizePort(uri.port)
13+
return runCatching {
14+
URI(
15+
"https",
16+
uri.userInfo,
17+
host,
18+
port,
19+
path,
20+
null,
21+
null,
22+
).toString()
23+
}.getOrDefault(trimmed)
24+
}
25+
26+
private fun normalizePath(rawPath: String?): String {
27+
if (rawPath.isNullOrBlank()) return ""
28+
var path: String = rawPath
29+
while (path.length > 1 && path.endsWith('/')) {
30+
path = path.dropLast(1)
31+
}
32+
return if (path == "/") "" else path
33+
}
34+
35+
private fun normalizePort(rawPort: Int): Int {
36+
if (rawPort == 80 || rawPort == 443) return -1
37+
return rawPort
38+
}
39+
}

‎src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,9 @@ class PipePipeBackupPersisterService {
5555

5656
private fun insertSubscriptions(userId: String, items: List<PipePipeBackupSubscriptionItem>): Int =
5757
items.sumOf { item ->
58+
val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(item.url)
5859
SubscriptionsTable.insertIgnore {
59-
it[SubscriptionsTable.userId] = userId; it[channelUrl] = item.url; it[name] = item.name; it[avatarUrl] = item.avatarUrl; it[subscribedAt] = System.currentTimeMillis()
60+
it[SubscriptionsTable.userId] = userId; it[channelUrl] = canonicalUrl; it[name] = item.name; it[avatarUrl] = item.avatarUrl; it[subscribedAt] = System.currentTimeMillis()
6061
}.insertedCount
6162
}
6263

‎src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,25 +21,27 @@ class SubscriptionsService {
2121
}
2222

2323
suspend fun add(userId: String, item: SubscriptionItem): SubscriptionItem {
24+
val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(item.channelUrl)
2425
val now = System.currentTimeMillis()
2526
DatabaseFactory.query {
2627
SubscriptionsTable.insert {
2728
it[SubscriptionsTable.userId] = userId
28-
it[channelUrl] = item.channelUrl
29+
it[channelUrl] = canonicalUrl
2930
it[name] = item.name
3031
it[avatarUrl] = item.avatarUrl
3132
it[subscribedAt] = now
3233
}
3334
}
34-
return item.copy(subscribedAt = now)
35+
return item.copy(channelUrl = canonicalUrl, subscribedAt = now)
3536
}
3637

3738
suspend fun delete(userId: String, channelUrl: String): Boolean = DatabaseFactory.query {
38-
SubscriptionsTable.deleteWhere { SubscriptionsTable.channelUrl eq channelUrl and (SubscriptionsTable.userId eq userId) } > 0
39+
val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(channelUrl)
40+
SubscriptionsTable.deleteWhere { SubscriptionsTable.channelUrl eq canonicalUrl and (SubscriptionsTable.userId eq userId) } > 0
3941
}
4042

4143
private fun ResultRow.toItem() = SubscriptionItem(
42-
channelUrl = this[SubscriptionsTable.channelUrl],
44+
channelUrl = ChannelUrlCanonicalizer.canonicalize(this[SubscriptionsTable.channelUrl]),
4345
name = this[SubscriptionsTable.name],
4446
avatarUrl = this[SubscriptionsTable.avatarUrl],
4547
subscribedAt = this[SubscriptionsTable.subscribedAt],

‎src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class YoutubeTakeoutImporterService(
2222
val existingSubsDeferred = async { subscriptionsService.getAll(userId).map { it.channelUrl }.toSet() }
2323
val existingPlaylistsDeferred = async { playlistService.getAll(userId) }
2424
val sourceMappingsDeferred = async { playlistKeyService.getMappings(userId).toMutableMap() }
25-
val existingSubs = existingSubsDeferred.await()
25+
val existingSubs = existingSubsDeferred.await().toMutableSet()
2626
val existingPlaylistRows = existingPlaylistsDeferred.await()
2727
val existingPlaylists = existingPlaylistRows.associateBy { it.name.lowercase() }
2828
val sourceMappings = sourceMappingsDeferred.await()
@@ -32,8 +32,10 @@ class YoutubeTakeoutImporterService(
3232
var subSkipped = 0
3333
if (plan.importSubscriptions) {
3434
parsed.subscriptions.forEach { item ->
35-
if (item.channelUrl in existingSubs) subSkipped += 1 else {
35+
val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(item.channelUrl)
36+
if (canonicalUrl in existingSubs) subSkipped += 1 else {
3637
subscriptionsService.add(userId, SubscriptionItem(item.channelUrl, item.name, item.avatarUrl))
38+
existingSubs += canonicalUrl
3739
subImported += 1
3840
}
3941
}

‎src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalMigrationTest.kt‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package dev.typetype.server
2+
3+
import dev.typetype.server.db.DatabaseSubscriptionsCanonicalMigration
4+
import dev.typetype.server.db.tables.SubscriptionsTable
5+
import org.jetbrains.exposed.v1.core.SortOrder
6+
import org.jetbrains.exposed.v1.core.eq
7+
import org.jetbrains.exposed.v1.jdbc.insert
8+
import org.jetbrains.exposed.v1.jdbc.selectAll
9+
import org.jetbrains.exposed.v1.jdbc.transactions.transaction
10+
import org.junit.jupiter.api.Assertions.assertEquals
11+
import org.junit.jupiter.api.BeforeAll
12+
import org.junit.jupiter.api.BeforeEach
13+
import org.junit.jupiter.api.Test
14+
15+
class SubscriptionsCanonicalMigrationTest {
16+
companion object {
17+
@BeforeAll
18+
@JvmStatic
19+
fun initDb() = TestDatabase.setup()
20+
}
21+
22+
@BeforeEach
23+
fun clean() {
24+
TestDatabase.truncateAll()
25+
}
26+
27+
@Test
28+
fun `migration canonicalizes and deduplicates subscriptions per user`() {
29+
transaction {
30+
SubscriptionsTable.insert {
31+
it[userId] = TEST_USER_ID
32+
it[channelUrl] = "https://www.youtube.com/channel/UC123"
33+
it[name] = "old"
34+
it[avatarUrl] = "a"
35+
it[subscribedAt] = 10L
36+
}
37+
SubscriptionsTable.insert {
38+
it[userId] = TEST_USER_ID
39+
it[channelUrl] = "http://WWW.YouTube.com/channel/UC123/?x=1#f"
40+
it[name] = "new"
41+
it[avatarUrl] = "b"
42+
it[subscribedAt] = 20L
43+
}
44+
SubscriptionsTable.insert {
45+
it[userId] = "other"
46+
it[channelUrl] = "http://WWW.YouTube.com/channel/UC123/?x=1#f"
47+
it[name] = "other"
48+
it[avatarUrl] = "c"
49+
it[subscribedAt] = 30L
50+
}
51+
DatabaseSubscriptionsCanonicalMigration.apply()
52+
val rows = SubscriptionsTable.selectAll()
53+
.where { SubscriptionsTable.userId eq TEST_USER_ID }
54+
.orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC)
55+
.toList()
56+
assertEquals(1, rows.size)
57+
assertEquals("https://www.youtube.com/channel/UC123", rows.first()[SubscriptionsTable.channelUrl])
58+
assertEquals("new", rows.first()[SubscriptionsTable.name])
59+
assertEquals(20L, rows.first()[SubscriptionsTable.subscribedAt])
60+
val otherRows = SubscriptionsTable.selectAll().where { SubscriptionsTable.userId eq "other" }.toList()
61+
assertEquals(1, otherRows.size)
62+
assertEquals("https://www.youtube.com/channel/UC123", otherRows.first()[SubscriptionsTable.channelUrl])
63+
}
64+
}
65+
}

‎src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package dev.typetype.server
2+
3+
import dev.typetype.server.models.SubscriptionItem
4+
import dev.typetype.server.routes.subscriptionsRoutes
5+
import dev.typetype.server.services.AuthService
6+
import dev.typetype.server.services.SubscriptionsService
7+
import io.ktor.client.request.delete
8+
import io.ktor.client.request.get
9+
import io.ktor.client.request.headers
10+
import io.ktor.client.request.post
11+
import io.ktor.client.request.setBody
12+
import io.ktor.client.statement.bodyAsText
13+
import io.ktor.http.ContentType
14+
import io.ktor.http.HttpHeaders
15+
import io.ktor.http.HttpStatusCode
16+
import io.ktor.serialization.kotlinx.json.json
17+
import io.ktor.server.application.install
18+
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
19+
import io.ktor.server.routing.routing
20+
import io.ktor.server.testing.ApplicationTestBuilder
21+
import io.ktor.server.testing.testApplication
22+
import org.junit.jupiter.api.Assertions.assertEquals
23+
import org.junit.jupiter.api.Assertions.assertTrue
24+
import org.junit.jupiter.api.BeforeAll
25+
import org.junit.jupiter.api.BeforeEach
26+
import org.junit.jupiter.api.Test
27+
28+
class SubscriptionsCanonicalizationRoutesTest {
29+
private val service = SubscriptionsService()
30+
private val auth = AuthService.fixed(TEST_USER_ID)
31+
32+
companion object {
33+
@BeforeAll
34+
@JvmStatic
35+
fun initDb() = TestDatabase.setup()
36+
}
37+
38+
@BeforeEach
39+
fun clean() {
40+
TestDatabase.truncateAll()
41+
}
42+
43+
private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication {
44+
application {
45+
install(ContentNegotiation) { json() }
46+
routing { subscriptionsRoutes(service, auth) }
47+
}
48+
block()
49+
}
50+
51+
@Test
52+
fun `POST and GET subscriptions return canonical channel url`() = withApp {
53+
val response = client.post("/subscriptions") {
54+
headers.append(HttpHeaders.Authorization, "Bearer test-jwt")
55+
headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString())
56+
setBody("""{"channelUrl":"http://WWW.YouTube.com/channel/UC123/?utm_source=x#frag","name":"Test","avatarUrl":""}""")
57+
}
58+
assertEquals(HttpStatusCode.Created, response.status)
59+
assertTrue(response.bodyAsText().contains("\"channelUrl\":\"https://www.youtube.com/channel/UC123\""))
60+
val listBody = client.get("/subscriptions") {
61+
headers.append(HttpHeaders.Authorization, "Bearer test-jwt")
62+
}.bodyAsText()
63+
assertTrue(listBody.contains("\"channelUrl\":\"https://www.youtube.com/channel/UC123\""))
64+
}
65+
66+
@Test
67+
fun `DELETE subscriptions canonicalizes channel url`() = withApp {
68+
service.add(
69+
TEST_USER_ID,
70+
SubscriptionItem(channelUrl = "https://www.youtube.com/channel/UC123", name = "Test", avatarUrl = ""),
71+
)
72+
val deleteResponse = client.delete("/subscriptions/http%3A%2F%2FWWW.YouTube.com%2Fchannel%2FUC123%2F%3Futm_source%3Dx%23frag") {
73+
headers.append(HttpHeaders.Authorization, "Bearer test-jwt")
74+
}
75+
assertEquals(HttpStatusCode.NoContent, deleteResponse.status)
76+
}
77+
}

0 commit comments

Comments
 (0)