fix: improve debug and ignore blank lines in excel merchant list

deploy-gcf.sh

@@ -18,7 +18,7 @@ PROJECT_ID="dash-wallet-firebase"
 RUNTIME="java17"
 ENTRY_POINT="org.dash.mobile.explore.sync.Function"
 MEMORY="1024MB"
-TIMEOUT="300s"
+TIMEOUT="600s"
 
 echo -e "${GREEN}=== Dash Explore Sync - Google Cloud Function Deployment ===${NC}\n"
 

src/main/kotlin/org/dash/mobile/explore/sync/MainApp.kt

@@ -30,7 +30,7 @@ fun main(args: Array<String>) {
                 println("$UPLOAD_ARG - force upload data to GC Storage")
                 println("$QUIET_ARG - quiet mode: no notifications are pushed to Slack")
                 println("$PROD_ARG - production mode: use production data sources/destinations")
-                println("$DEBUG_ARG - output to CSV files for unit tests")
+                println("$DEBUG_ARG - output to CSV files for unit tests, BODY logging")
                 exitProcess(1)
             }
         }

src/main/kotlin/org/dash/mobile/explore/sync/SyncProcessor.kt

@@ -40,7 +40,7 @@ import java.util.zip.CheckedInputStream
 class SyncProcessor(private val mode: OperationMode, private val debug: Boolean = false) {
     companion object {
         const val CURRENT_VERSION = 4
-        const val BUILD = 5
+        const val BUILD = 6
     }
 
     private val logger = LoggerFactory.getLogger(SyncProcessor::class.java)!!
@@ -139,10 +139,10 @@ class SyncProcessor(private val mode: OperationMode, private val debug: Boolean
 
     @Throws(SQLException::class)
     private suspend fun importData(dbFile: File, locationsDbFile: File) {
-        val ctxDataSource = CTXSpendDataSource(slackMessenger)
+        val ctxDataSource = CTXSpendDataSource(slackMessenger, debug)
         val ctxData = ctxDataSource.getDataList()
         val ctxReport = ctxDataSource.getReport()
-        val piggyCardsDataSource = PiggyCardsDataSource(slackMessenger, mode)
+        val piggyCardsDataSource = PiggyCardsDataSource(slackMessenger, mode, debug)
         val piggyCardsData = piggyCardsDataSource.getDataList()
         val piggyCardsReport = piggyCardsDataSource.getReport()
         val report = SyncReport(listOf(ctxReport, piggyCardsReport))
@@ -156,7 +156,7 @@ class SyncProcessor(private val mode: OperationMode, private val debug: Boolean
         val combinedMerchants = try {
             // merchant table
             var prepStatement = dbConnection.prepareStatement(MerchantData.INSERT_STATEMENT)
-            val dcgDataFlow = DCGDataSource(mode != OperationMode.PRODUCTION, slackMessenger).getData(prepStatement)
+            val dcgDataFlow = DCGDataSource(mode != OperationMode.PRODUCTION, slackMessenger, debug).getData(prepStatement)
             val merger = MerchantLocationMerger(debug)
             val combinedMerchants = merger.combineMerchants(listOf(ctxData, piggyCardsData))
             logger.info("Duplicate locations: ${combinedMerchants.matchInfo.size}")
@@ -187,7 +187,7 @@ class SyncProcessor(private val mode: OperationMode, private val debug: Boolean
 
             // atm table
             prepStatement = dbConnection.prepareStatement(AtmLocation.INSERT_STATEMENT)
-            val coinAtmRadarDataFlow = CoinAtmRadarDataSource(slackMessenger).getData(prepStatement)
+            val coinAtmRadarDataFlow = CoinAtmRadarDataSource(slackMessenger, debug).getData(prepStatement)
             val atmDataFlow = flowOf(coinAtmRadarDataFlow).flattenConcat()
             syncData(atmDataFlow, prepStatement)
             combinedMerchants

src/main/kotlin/org/dash/mobile/explore/sync/process/CTXSpendDataSource.kt

@@ -27,8 +27,8 @@ private const val BASE_URL = "https://spend.ctx.com/"
 /**
  * Import data from CTXSpend API
  */
-class CTXSpendDataSource(slackMessenger: SlackMessenger) :
-    DataSource<MerchantData>(slackMessenger) {
+class CTXSpendDataSource(slackMessenger: SlackMessenger, debugMode: Boolean) :
+    DataSource<MerchantData>(slackMessenger, debugMode) {
     override val logger = LoggerFactory.getLogger(CTXSpendDataSource::class.java)!!
     val merchantList = hashSetOf<String>()
     var dataSourceReport: DataSourceReport? = null
@@ -74,7 +74,7 @@ class CTXSpendDataSource(slackMessenger: SlackMessenger) :
             .readTimeout(30, TimeUnit.SECONDS)
             .also { client ->
                 val logging = HttpLoggingInterceptor { message -> println(message) }
-                logging.level = HttpLoggingInterceptor.Level.HEADERS
+                logging.level = loggingLevel
                 logging.redactHeader("Authorization")
                 client.addInterceptor(logging)
             }

src/main/kotlin/org/dash/mobile/explore/sync/process/CoinAtmRadarDataSource.kt

@@ -20,7 +20,7 @@ import java.security.MessageDigest
 /**
  * Import data from CoinAtmRadar API
  */
-class CoinAtmRadarDataSource(slackMessenger: SlackMessenger) : DataSource<AtmLocation>(slackMessenger) {
+class CoinAtmRadarDataSource(slackMessenger: SlackMessenger, debugMode: Boolean) : DataSource<AtmLocation>(slackMessenger, debugMode) {
     companion object {
         private const val BASE_URL = "https://coinatmradar.com/ext_api/"
     }

src/main/kotlin/org/dash/mobile/explore/sync/process/DCGDataSource.kt

@@ -23,8 +23,8 @@ private const val SPREADSHEET_ID = "1YU5UShf5ruTZKJxglP36h-87W02bsDY3L5MmpYjFCGA
 /**
  * Import data from Google Sheet: https://docs.google.com/spreadsheets/d/1YU5UShf5ruTZKJxglP36h-87W02bsDY3L5MmpYjFCGA
  */
-class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackMessenger) :
-    DataSource<MerchantData>(slackMessenger) {
+class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackMessenger, debugMode: Boolean) :
+    DataSource<MerchantData>(slackMessenger, debugMode) {
 
     override val logger = LoggerFactory.getLogger(DCGDataSource::class.java)!!
 
@@ -97,6 +97,7 @@ class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackM
         val headers = mutableListOf<String>()
 
         var totalRecords = 0
+        val merchantNames = hashSetOf<String>()
         for (rowIndex in values.indices) {
             val rowData = values[rowIndex]
             if (rowIndex == 0) {
@@ -115,13 +116,17 @@ class DCGDataSource(private val useTestnetSheet: Boolean, slackMessenger: SlackM
                 val merchant = convertFromValues(rowData)
                 if (merchant != null) {
                     totalRecords++
+                    merchant.name?.let {
+                        merchantNames.add(it)
+                    }
                     emit(merchant)
                 } else {
-                    break
+                    logger.info("Skipping empty row at index $rowIndex")
+                    continue
                 }
             }
         }
-        slackMessenger.postSlackMessage("DCG Merchants $totalRecords records", logger)
+        slackMessenger.postSlackMessage("DCG Merchants $totalRecords location records (merchants: ${merchantNames.size})", logger)
     }
 
     private fun convertFromValues(rowData: List<Any>): MerchantData? {

src/main/kotlin/org/dash/mobile/explore/sync/process/DataSource.kt

@@ -7,22 +7,24 @@ import com.google.gson.reflect.TypeToken
 import com.google.gson.stream.JsonReader
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.flow.transform
 import kotlinx.coroutines.withContext
+import okhttp3.logging.HttpLoggingInterceptor
 import org.dash.mobile.explore.sync.notice
 import org.dash.mobile.explore.sync.process.data.Data
-import org.dash.mobile.explore.sync.process.data.MerchantData
 import org.dash.mobile.explore.sync.slack.SlackMessenger
 import org.slf4j.Logger
 import java.io.FileNotFoundException
 import java.io.InputStreamReader
 import java.sql.PreparedStatement
 import java.util.Properties
 
-abstract class DataSource<T>(val slackMessenger: SlackMessenger) where T : Data {
+abstract class DataSource<T>(val slackMessenger: SlackMessenger, val debugMode: Boolean) where T : Data {
 
     private val usStatesAbbrMap: Map<String?, String?>
+    protected val loggingLevel = if (debugMode)
+        HttpLoggingInterceptor.Level.BODY
+    else HttpLoggingInterceptor.Level.HEADERS
 
     init {
         val gsonReader = Gson()

src/main/kotlin/org/dash/mobile/explore/sync/process/PiggyCardsDataSource.kt

@@ -28,8 +28,8 @@ private const val BASE_URL = PROD_BASE_URL
 /**
  * Import data from PiggyCards API
  */
-class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: OperationMode) :
-    DataSource<MerchantData>(slackMessenger) {
+class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: OperationMode, debugMode: Boolean) :
+    DataSource<MerchantData>(slackMessenger, debugMode) {
         companion object {
             const val SERVICE_FEE = 150 // 1.5% for CurPay
         }
@@ -80,6 +80,27 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope
             }
         }
 
+        /**
+         * Location
+         * {
+         *      "name":"Burger King",
+         *      "latitude":49.666313,
+         *      "longitude":-112.793823,
+         *      "street_number":"2416",
+         *      "street":"Fairway Plaza Rd S",
+         *      "country":"CA",
+         *      "city":"Lethbridge",
+         *      "state":"AB",
+         *      "zip":"Unknown",
+         *      "opening_hours":"Unknown",
+         *      "phone":"(403) 380-4771",
+         *      "shop":"Unknown",
+         *      "website":"Unknown",
+         *      "wheelchair":"Unknown"
+         * }
+         *
+         */
+
         data class Location(
             val name: String,
             val latitude: Double,
@@ -89,6 +110,7 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope
             val city: String,
             val state: String,
             val zip: String,
+            val country: String,
             @SerializedName("opening_hours") val openingHours: String,
             val phone: String,
             val shop: String,
@@ -157,7 +179,7 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope
             .addInterceptor(PiggyCardsHeadersInterceptor() { token })
             .also { client ->
                 val logging = HttpLoggingInterceptor { message -> println(message) }
-                logging.level = HttpLoggingInterceptor.Level.HEADERS
+                logging.level = loggingLevel
                 logging.redactHeader("Authorization")
                 client.addInterceptor(logging)
             }
@@ -232,10 +254,10 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope
                         if (giftCard.name.lowercase().contains("(instant delivery)")) {
                             immediateDeliveryCards.add(giftCard)
                         }
-                        if (immediateDeliveryCards.isNotEmpty()) {
-                            // add rest of fixed cards
-                            immediateDeliveryCards.addAll(giftCards.filter { it.priceType == "Fixed" && !it.name.contains("(instant delivery)")} )
-                        }
+                    }
+                    if (giftCards != null && immediateDeliveryCards.isNotEmpty()) {
+                        // add rest of fixed cards
+                        immediateDeliveryCards.addAll(giftCards.filter { it.priceType == "Fixed" && !it.name.contains("(instant delivery)")} )
                     }
 
                     // choose the first non-fixed card if available, otherwise the first card
@@ -266,7 +288,7 @@ class PiggyCardsDataSource(slackMessenger: SlackMessenger, private val mode: Ope
 
                             var locationsAdded = 0
                             locations.forEach { location ->
-                                if (isValidLocation("physical", location)) {
+                                if (isValidLocation("physical", location) && location.country == country) {
                                     val merchantWithLocation = merchantData.copy(
                                         address1 = createAddress(location),
                                         city = location.city,