From 327e3900d4893d950cbb595b3098f170b865f30f Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Tue, 13 Jan 2026 14:32:36 -0800 Subject: [PATCH 1/7] fix: raise timeout to 600s (10 min) for Google Functions --- deploy-gcf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-gcf.sh b/deploy-gcf.sh index 44a8de2..1974a01 100755 --- a/deploy-gcf.sh +++ b/deploy-gcf.sh @@ -18,7 +18,7 @@ PROJECT_ID="dash-wallet-firebase" RUNTIME="java17" ENTRY_POINT="org.dash.mobile.explore.sync.Function" MEMORY="1024MB" -TIMEOUT="300s" +TIMEOUT="600s" echo -e "${GREEN}=== Dash Explore Sync - Google Cloud Function Deployment ===${NC}\n" From 0151fe78e494c37db2c42369053b6889a7f7ef0a Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Tue, 13 Jan 2026 20:26:02 -0800 Subject: [PATCH 2/7] feat: use -debug to add http logging (body) --- .../kotlin/org/dash/mobile/explore/sync/MainApp.kt | 2 +- .../explore/sync/process/CTXSpendDataSource.kt | 6 +++--- .../explore/sync/process/CoinAtmRadarDataSource.kt | 2 +- .../mobile/explore/sync/process/DCGDataSource.kt | 4 ++-- .../dash/mobile/explore/sync/process/DataSource.kt | 8 +++++--- .../explore/sync/process/PiggyCardsDataSource.kt | 14 +++++++------- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/MainApp.kt b/src/main/kotlin/org/dash/mobile/explore/sync/MainApp.kt index 59b3560..361f808 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/MainApp.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/MainApp.kt @@ -30,7 +30,7 @@ fun main(args: Array) { println("$UPLOAD_ARG - force upload data to GC Storage") println("$QUIET_ARG - quiet mode: no notifications are pushed to Slack") println("$PROD_ARG - production mode: use production data sources/destinations") - println("$DEBUG_ARG - output to CSV files for unit tests") + println("$DEBUG_ARG - output to CSV files for unit tests, BODY logging") exitProcess(1) } } diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/process/CTXSpendDataSource.kt b/src/main/kotlin/org/dash/mobile/explore/sync/process/CTXSpendDataSource.kt index 7b8147a..290e1a0 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/process/CTXSpendDataSource.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/process/CTXSpendDataSource.kt @@ -27,8 +27,8 @@ private const val BASE_URL = "https://spend.ctx.com/" /** * Import data from CTXSpend API */ -class CTXSpendDataSource(slackMessenger: SlackMessenger) : - DataSource(slackMessenger) { +class CTXSpendDataSource(slackMessenger: SlackMessenger, debugMode: Boolean) : + DataSource(slackMessenger, debugMode) { override val logger = LoggerFactory.getLogger(CTXSpendDataSource::class.java)!! val merchantList = hashSetOf() var dataSourceReport: DataSourceReport? = null @@ -74,7 +74,7 @@ class CTXSpendDataSource(slackMessenger: SlackMessenger) : .readTimeout(30, TimeUnit.SECONDS) .also { client -> val logging = HttpLoggingInterceptor { message -> println(message) } - logging.level = HttpLoggingInterceptor.Level.HEADERS + logging.level = loggingLevel logging.redactHeader("Authorization") client.addInterceptor(logging) } diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/process/CoinAtmRadarDataSource.kt b/src/main/kotlin/org/dash/mobile/explore/sync/process/CoinAtmRadarDataSource.kt index 0a30185..62c71ca 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/process/CoinAtmRadarDataSource.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/process/CoinAtmRadarDataSource.kt @@ -20,7 +20,7 @@ import java.security.MessageDigest /** * Import data from CoinAtmRadar API */ -class CoinAtmRadarDataSource(slackMessenger: SlackMessenger) : DataSource(slackMessenger) { +class CoinAtmRadarDataSource(slackMessenger: SlackMessenger, debugMode: Boolean) : DataSource(slackMessenger, debugMode) { companion object { private const val BASE_URL = "https://coinatmradar.com/ext_api/" } diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt b/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt index 28c162f..e88e30f 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt @@ -23,8 +23,8 @@ private const val SPREADSHEET_ID = "1YU5UShf5ruTZKJxglP36h-87W02bsDY3L5MmpYjFCGA /** * Import data from Google Sheet: https://docs.google.com/spreadsheets/d/1YU5UShf5ruTZKJxglP36h-87W02bsDY3L5MmpYjFCGA */ -class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackMessenger) : - DataSource(slackMessenger) { +class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackMessenger, debugMode: Boolean) : + DataSource(slackMessenger, debugMode) { override val logger = LoggerFactory.getLogger(DCGDataSource::class.java)!! diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/process/DataSource.kt b/src/main/kotlin/org/dash/mobile/explore/sync/process/DataSource.kt index e327824..f3db4f6 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/process/DataSource.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/process/DataSource.kt @@ -7,12 +7,11 @@ import com.google.gson.reflect.TypeToken import com.google.gson.stream.JsonReader import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.transform import kotlinx.coroutines.withContext +import okhttp3.logging.HttpLoggingInterceptor import org.dash.mobile.explore.sync.notice import org.dash.mobile.explore.sync.process.data.Data -import org.dash.mobile.explore.sync.process.data.MerchantData import org.dash.mobile.explore.sync.slack.SlackMessenger import org.slf4j.Logger import java.io.FileNotFoundException @@ -20,9 +19,12 @@ import java.io.InputStreamReader import java.sql.PreparedStatement import java.util.Properties -abstract class DataSource(val slackMessenger: SlackMessenger) where T : Data { +abstract class DataSource(val slackMessenger: SlackMessenger, val debugMode: Boolean) where T : Data { private val usStatesAbbrMap: Map + protected val loggingLevel = if (debugMode) + HttpLoggingInterceptor.Level.BODY + else HttpLoggingInterceptor.Level.HEADERS init { val gsonReader = Gson() diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt b/src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt index 45ff218..806ef01 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt @@ -28,8 +28,8 @@ private const val BASE_URL = PROD_BASE_URL /** * Import data from PiggyCards API */ -class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: OperationMode) : - DataSource(slackMessenger) { +class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: OperationMode, debugMode: Boolean) : + DataSource(slackMessenger, debugMode) { companion object { const val SERVICE_FEE = 150 // 1.5% for CurPay } @@ -157,7 +157,7 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope .addInterceptor(PiggyCardsHeadersInterceptor() { token }) .also { client -> val logging = HttpLoggingInterceptor { message -> println(message) } - logging.level = HttpLoggingInterceptor.Level.HEADERS + logging.level = loggingLevel logging.redactHeader("Authorization") client.addInterceptor(logging) } @@ -232,10 +232,10 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope if (giftCard.name.lowercase().contains("(instant delivery)")) { immediateDeliveryCards.add(giftCard) } - if (immediateDeliveryCards.isNotEmpty()) { - // add rest of fixed cards - immediateDeliveryCards.addAll(giftCards.filter { it.priceType == "Fixed" && !it.name.contains("(instant delivery)")} ) - } + } + if (giftCards != null && immediateDeliveryCards.isNotEmpty()) { + // add rest of fixed cards + immediateDeliveryCards.addAll(giftCards.filter { it.priceType == "Fixed" && !it.name.contains("(instant delivery)")} ) } // choose the first non-fixed card if available, otherwise the first card From 9bfc5292eb8b77ebba8944b61f24fc829fef418f Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Tue, 27 Jan 2026 22:19:55 -0800 Subject: [PATCH 3/7] feat: add country to PiggyCards locations and filter accordingly --- .../sync/process/PiggyCardsDataSource.kt | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt b/src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt index 806ef01..efde630 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt @@ -80,6 +80,27 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope } } + /** + * Location + * { + * "name":"Burger King", + * "latitude":49.666313, + * "longitude":-112.793823, + * "street_number":"2416", + * "street":"Fairway Plaza Rd S", + * "country":"CA", + * "city":"Lethbridge", + * "state":"AB", + * "zip":"Unknown", + * "opening_hours":"Unknown", + * "phone":"(403) 380-4771", + * "shop":"Unknown", + * "website":"Unknown", + * "wheelchair":"Unknown" + * } + * + */ + data class Location( val name: String, val latitude: Double, @@ -89,6 +110,7 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope val city: String, val state: String, val zip: String, + val country: String, @SerializedName("opening_hours") val openingHours: String, val phone: String, val shop: String, @@ -266,7 +288,7 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope var locationsAdded = 0 locations.forEach { location -> - if (isValidLocation("physical", location)) { + if (isValidLocation("physical", location) && location.country == country) { val merchantWithLocation = merchantData.copy( address1 = createAddress(location), city = location.city, From 13a6a5ef3b146d28fcc38cb560b1b8610b3a25ce Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Tue, 27 Jan 2026 22:20:15 -0800 Subject: [PATCH 4/7] fix: skip blank lines in DCG merchants list --- .../org/dash/mobile/explore/sync/process/DCGDataSource.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt b/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt index e88e30f..5048d8d 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt @@ -117,7 +117,8 @@ class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackM totalRecords++ emit(merchant) } else { - break + logger.info("Skipping empty row at index $rowIndex") + continue } } } From dba1d1532bea132a2f05b060287140a5969ae964 Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Tue, 27 Jan 2026 22:20:56 -0800 Subject: [PATCH 5/7] feat: count merchant names in DCG merchant list --- .../org/dash/mobile/explore/sync/process/DCGDataSource.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt b/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt index 5048d8d..bbb71fd 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt @@ -97,6 +97,7 @@ class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackM val headers = mutableListOf() var totalRecords = 0 + val merchantNames = hashSetOf() for (rowIndex in values.indices) { val rowData = values[rowIndex] if (rowIndex == 0) { @@ -115,6 +116,9 @@ class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackM val merchant = convertFromValues(rowData) if (merchant != null) { totalRecords++ + merchant.name?.let { + merchantNames.add(it) + } emit(merchant) } else { logger.info("Skipping empty row at index $rowIndex") @@ -122,7 +126,7 @@ class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackM } } } - slackMessenger.postSlackMessage("DCG Merchants $totalRecords records", logger) + slackMessenger.postSlackMessage("DCG Merchants $totalRecords location records (merchants: ${merchantNames.size})", logger) } private fun convertFromValues(rowData: List): MerchantData? { From 61d267107b6754925633e785951dcbebcf25e52e Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Tue, 27 Jan 2026 22:21:37 -0800 Subject: [PATCH 6/7] fix: pass debug parameter to DataSource objects --- .../kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt b/src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt index 16691e5..350a90c 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt @@ -139,10 +139,10 @@ class SyncProcessor(private val mode: OperationMode, private val debug: Boolean @Throws(SQLException::class) private suspend fun importData(dbFile: File, locationsDbFile: File) { - val ctxDataSource = CTXSpendDataSource(slackMessenger) + val ctxDataSource = CTXSpendDataSource(slackMessenger, debug) val ctxData = ctxDataSource.getDataList() val ctxReport = ctxDataSource.getReport() - val piggyCardsDataSource = PiggyCardsDataSource(slackMessenger, mode) + val piggyCardsDataSource = PiggyCardsDataSource(slackMessenger, mode, debug) val piggyCardsData = piggyCardsDataSource.getDataList() val piggyCardsReport = piggyCardsDataSource.getReport() val report = SyncReport(listOf(ctxReport, piggyCardsReport)) @@ -156,7 +156,7 @@ class SyncProcessor(private val mode: OperationMode, private val debug: Boolean val combinedMerchants = try { // merchant table var prepStatement = dbConnection.prepareStatement(MerchantData.INSERT_STATEMENT) - val dcgDataFlow = DCGDataSource(mode != OperationMode.PRODUCTION, slackMessenger).getData(prepStatement) + val dcgDataFlow = DCGDataSource(mode != OperationMode.PRODUCTION, slackMessenger, debug).getData(prepStatement) val merger = MerchantLocationMerger(debug) val combinedMerchants = merger.combineMerchants(listOf(ctxData, piggyCardsData)) logger.info("Duplicate locations: ${combinedMerchants.matchInfo.size}") @@ -187,7 +187,7 @@ class SyncProcessor(private val mode: OperationMode, private val debug: Boolean // atm table prepStatement = dbConnection.prepareStatement(AtmLocation.INSERT_STATEMENT) - val coinAtmRadarDataFlow = CoinAtmRadarDataSource(slackMessenger).getData(prepStatement) + val coinAtmRadarDataFlow = CoinAtmRadarDataSource(slackMessenger, debug).getData(prepStatement) val atmDataFlow = flowOf(coinAtmRadarDataFlow).flattenConcat() syncData(atmDataFlow, prepStatement) combinedMerchants From 283052ef629eeac5ff520a0da7008c531e2eea6f Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Thu, 29 Jan 2026 16:46:40 -0800 Subject: [PATCH 7/7] fix: bump build number to 6 --- src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt b/src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt index 350a90c..08d0253 100644 --- a/src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt +++ b/src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt @@ -40,7 +40,7 @@ import java.util.zip.CheckedInputStream class SyncProcessor(private val mode: OperationMode, private val debug: Boolean = false) { companion object { const val CURRENT_VERSION = 4 - const val BUILD = 5 + const val BUILD = 6 } private val logger = LoggerFactory.getLogger(SyncProcessor::class.java)!!