stats.service.ts

import { IDailyHMT, NETWORKS, StatisticsClient } from '@human-protocol/sdk';
import { HttpService } from '@nestjs/axios';
import { Cache, CACHE_MANAGER } from '@nestjs/cache-manager';
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { Cron, SchedulerRegistry } from '@nestjs/schedule';
import { AxiosError } from 'axios';
import { CronJob } from 'cron';
import dayjs from 'dayjs';
import { lastValueFrom } from 'rxjs';
import {
  EnvironmentConfigService,
  HCAPTCHA_STATS_API_START_DATE,
  HCAPTCHA_STATS_START_DATE,
  HMT_STATS_START_DATE,
} from '../../common/config/env-config.service';
import {
  HCAPTCHA_PREFIX,
  HMT_PREFIX,
  RedisConfigService,
} from '../../common/config/redis-config.service';
import * as httpUtils from '../../common/utils/http';
import logger from '../../logger';
import { NetworksService } from '../networks/networks.service';
import { StorageService } from '../storage/storage.service';
import { HcaptchaDailyStats, HcaptchaStats } from './dto/hcaptcha.dto';
import { HmtGeneralStatsDto } from './dto/hmt-general-stats.dto';
import { HmtDailyStatsData } from './dto/hmt.dto';
import { CachedHMTData } from './stats.interface';

type HcaptchaDailyStat = {
  solved: number;
  served?: number;
};

@Injectable()
export class StatsService implements OnModuleInit {
  private readonly logger = logger.child({ context: StatsService.name });

  constructor(
    @Inject(CACHE_MANAGER) private readonly cacheManager: Cache,
    private readonly redisConfigService: RedisConfigService,
    private readonly networksService: NetworksService,
    private readonly envConfigService: EnvironmentConfigService,
    private readonly httpService: HttpService,
    private readonly storageService: StorageService,
    private readonly schedulerRegistry: SchedulerRegistry,
  ) {
    if (this.envConfigService.hCaptchaStatsEnabled === true) {
      const job = new CronJob('*/15 * * * *', () => {
        this.fetchTodayHcaptchaStats();
      });

      this.schedulerRegistry.addCronJob('fetchTodayHcaptchaStats', job);
      job.start();
    }
  }

  async onModuleInit() {
    const isHistoricalDataFetched = await this.isHistoricalDataFetched();
    const isHmtGeneralStatsFetched = await this.isHmtGeneralStatsFetched();
    const isHmtDailyStatsFetched = await this.isHmtDailyStatsFetched();

    if (
      this.envConfigService.hCaptchaStatsEnabled === true &&
      !isHistoricalDataFetched
    ) {
      await this.fetchHistoricalHcaptchaStats();
    }
    if (!isHmtGeneralStatsFetched) {
      await this.fetchHmtGeneralStats();
    }
    if (!isHmtDailyStatsFetched) {
      await this.fetchHistoricalHmtStats();
    }
  }

  private async isHistoricalDataFetched(): Promise<boolean> {
    const data = await this.cacheManager.get<HcaptchaDailyStats>(
      `${HCAPTCHA_PREFIX}${HCAPTCHA_STATS_START_DATE}`,
    );
    return !!data;
  }

  private async fetchHistoricalHcaptchaStats(): Promise<void> {
    let startDate = dayjs(HCAPTCHA_STATS_API_START_DATE);

    this.logger.debug('Fetching historical hCaptcha stats', {
      startDate,
    });

    try {
      const currentDate = dayjs();
      const dates = [];

      while (startDate <= currentDate) {
        const from = startDate.startOf('month').format('YYYY-MM-DD');
        const to = startDate.endOf('month').format('YYYY-MM-DD');

        dates.push({ from, to });

        startDate = startDate.add(1, 'month');
      }

      let hCaptchaStats: HcaptchaDailyStat[][];

      try {
        const statsFile = await this.storageService.downloadFile(
          this.envConfigService.hCaptchaStatsFile,
        );
        hCaptchaStats = JSON.parse(statsFile.toString());
      } catch (error) {
        this.logger.error('Error while getting hCaptcha stats file', error);
        hCaptchaStats = [];
      }

      for (const range of dates) {
        const { data } = await lastValueFrom(
          this.httpService.get(this.envConfigService.hCaptchaStatsSource, {
            params: {
              start_date: range.from,
              end_date: range.to,
              api_key: this.envConfigService.hCaptchaApiKey,
            },
          }),
        );
        hCaptchaStats.push(data);
      }

      for (const monthData of hCaptchaStats) {
        for (const [date, value] of Object.entries(monthData)) {
          const multiplier = date <= '2022-11-30' ? 18 : 9;
          if (value.served) delete value.served;
          value.solved *= multiplier;
          if (date !== 'total') {
            await this.cacheManager.set(`${HCAPTCHA_PREFIX}${date}`, value);
          } else {
            const dates = Object.keys(monthData).filter(
              (key) => key !== 'total',
            );
            if (dates.length > 0) {
              const month = dayjs(dates[0]).format('YYYY-MM');
              await this.cacheManager.set(`${HCAPTCHA_PREFIX}${month}`, value);
            }
          }
        }
      }
    } catch (error) {
      let formattedError = error;
      if (error instanceof AxiosError) {
        formattedError = httpUtils.formatAxiosError(error);
      }
      this.logger.error('Failed to fetch historical hCaptcha stats', {
        startDate,
        error: formattedError,
      });
    }
  }

  private async isHmtGeneralStatsFetched(): Promise<boolean> {
    const data = await this.cacheManager.get<HmtGeneralStatsDto>(
      this.redisConfigService.hmtGeneralStatsCacheKey,
    );
    return !!data;
  }

  async fetchTodayHcaptchaStats() {
    const today = dayjs().format('YYYY-MM-DD');
    const from = today;
    const to = today;

    this.logger.debug('Fetching hCaptcha stats for today', { from, to });

    try {
      const { data } = await lastValueFrom(
        this.httpService.get(this.envConfigService.hCaptchaStatsSource, {
          params: {
            start_date: from,
            end_date: to,
            api_key: this.envConfigService.hCaptchaApiKey,
          },
        }),
      );

      const multiplier = today <= '2022-11-30' ? 18 : 9;
      const stats = data[today];
      if (stats) {
        if (stats.served) delete stats.served;
        stats.solved *= multiplier;
        await this.cacheManager.set(`${HCAPTCHA_PREFIX}${today}`, stats);
      }

      const currentMonth = dayjs().format('YYYY-MM');
      const daysInMonth = dayjs().daysInMonth();

      const dates = Array.from(
        { length: daysInMonth },
        (_, i) => `${currentMonth}-${String(i + 1).padStart(2, '0')}`,
      );

      const aggregatedStats = await Promise.all(
        dates.map(async (date) => {
          const dailyStats: HcaptchaDailyStats = await this.cacheManager.get(
            `${HCAPTCHA_PREFIX}${date}`,
          );
          return dailyStats || { solved: 0 };
        }),
      ).then((statsArray) =>
        statsArray.reduce(
          (acc, stats) => {
            acc.solved += stats.solved;
            return acc;
          },
          { solved: 0 },
        ),
      );

      await this.cacheManager.set(
        `${HCAPTCHA_PREFIX}${currentMonth}`,
        aggregatedStats,
      );
    } catch (error) {
      let formattedError = error;
      if (error instanceof AxiosError) {
        formattedError = httpUtils.formatAxiosError(error);
      }
      this.logger.error('Failed to fetch todays hCaptcha stats', {
        today,
        from,
        to,
        error: formattedError,
      });
    }
  }

  @Cron('*/15 * * * *')
  async fetchHmtGeneralStats() {
    this.logger.debug('Fetching HMT general stats across multiple networks');

    try {
      const aggregatedStats: HmtGeneralStatsDto = {
        totalHolders: 0,
        totalTransactions: 0,
      };
      const operatingNetworks =
        await this.networksService.getOperatingNetworks();
      for (const network of operatingNetworks) {
        const statisticsClient = new StatisticsClient(NETWORKS[network]);
        const generalStats = await statisticsClient.getHMTStatistics();
        aggregatedStats.totalHolders += generalStats.totalHolders;
        aggregatedStats.totalTransactions += generalStats.totalTransferCount;
      }

      await this.cacheManager.set(
        this.redisConfigService.hmtGeneralStatsCacheKey,
        aggregatedStats,
      );
    } catch (error) {
      let formattedError = error;
      if (error instanceof AxiosError) {
        formattedError = httpUtils.formatAxiosError(error);
      }
      this.logger.error('Failed to fetch HMT general stats', formattedError);
    }
  }

  private async isHmtDailyStatsFetched(): Promise<boolean> {
    const data = await this.cacheManager.get<IDailyHMT>(
      `${HMT_PREFIX}${HMT_STATS_START_DATE}`,
    );
    return !!data;
  }

  private async fetchHistoricalHmtStats(): Promise<void> {
    const startDate = dayjs(HMT_STATS_START_DATE);
    await this.fetchAndCacheHmtDailyStats(startDate.format('YYYY-MM-DD'));
  }

  @Cron('*/15 * * * *')
  async fetchHmtDailyStats() {
    const currentDate = dayjs().format('YYYY-MM-DD');
    await this.fetchAndCacheHmtDailyStats(currentDate);
  }

  private async fetchAndCacheHmtDailyStats(date: string) {
    const from = new Date(date);
    const to = new Date(dayjs().format('YYYY-MM-DD'));

    try {
      const dailyData: Record<string, CachedHMTData> = {};
      const monthlyData: Record<string, CachedHMTData> = {};

      const operatingNetworks =
        await this.networksService.getOperatingNetworks();

      // Fetch daily data for each network
      await Promise.all(
        operatingNetworks.map(async (network) => {
          const statisticsClient = new StatisticsClient(NETWORKS[network]);
          let skip = 0;
          let fetchedRecords: IDailyHMT[] = [];

          do {
            fetchedRecords = await statisticsClient.getHMTDailyData({
              from,
              to,
              first: 1000, // Max subgraph query size
              skip,
            });

            for (const record of fetchedRecords) {
              const dailyCacheKey = `${HMT_PREFIX}${
                new Date(record.timestamp).toISOString().split('T')[0]
              }`;

              // Sum daily values
              if (!dailyData[dailyCacheKey]) {
                dailyData[dailyCacheKey] = {
                  totalTransactionAmount: '0',
                  totalTransactionCount: 0,
                  dailyUniqueSenders: 0,
                  dailyUniqueReceivers: 0,
                };
              }

              dailyData[dailyCacheKey].totalTransactionAmount = (
                BigInt(dailyData[dailyCacheKey].totalTransactionAmount) +
                record.totalTransactionAmount
              ).toString();
              dailyData[dailyCacheKey].totalTransactionCount +=
                record.totalTransactionCount;
              dailyData[dailyCacheKey].dailyUniqueSenders +=
                record.dailyUniqueSenders;
              dailyData[dailyCacheKey].dailyUniqueReceivers +=
                record.dailyUniqueReceivers;

              // Sum monthly values
              const month = dayjs(record.timestamp).format('YYYY-MM');
              if (!monthlyData[month]) {
                monthlyData[month] = {
                  totalTransactionAmount: '0',
                  totalTransactionCount: 0,
                  dailyUniqueSenders: 0,
                  dailyUniqueReceivers: 0,
                };
              }

              monthlyData[month].totalTransactionAmount = (
                BigInt(monthlyData[month].totalTransactionAmount) +
                record.totalTransactionAmount
              ).toString();
              monthlyData[month].totalTransactionCount +=
                record.totalTransactionCount;
              monthlyData[month].dailyUniqueSenders +=
                record.dailyUniqueSenders;
              monthlyData[month].dailyUniqueReceivers +=
                record.dailyUniqueReceivers;
            }

            skip += 1000;
          } while (fetchedRecords.length === 1000);
        }),
      );

      // Store daily records
      for (const [dailyCacheKey, stats] of Object.entries(dailyData)) {
        await this.cacheManager.set(dailyCacheKey, stats);
      }

      // Store monthly records
      for (const [month, stats] of Object.entries(monthlyData)) {
        const monthlyCacheKey = `${HMT_PREFIX}${month}`;
        await this.cacheManager.set(monthlyCacheKey, stats);
      }
    } catch (error) {
      let formattedError = error;
      if (error instanceof AxiosError) {
        formattedError = httpUtils.formatAxiosError(error);
      }
      this.logger.error('Failed to fetch HMT daily status', {
        from,
        to,
        error: formattedError,
      });
    }
  }

  async hmtPrice(): Promise<number> {
    const cachedHmtPrice: number = await this.cacheManager.get<number>(
      this.redisConfigService.hmtPriceCacheKey,
    );
    if (cachedHmtPrice) {
      return cachedHmtPrice;
    }

    const headers = this.envConfigService.hmtPriceSourceApiKey
      ? { 'x-cg-demo-api-key': this.envConfigService.hmtPriceSourceApiKey }
      : {};

    const { data } = await lastValueFrom(
      this.httpService.get(this.envConfigService.hmtPriceSource, { headers }),
    );

    let hmtPrice: number;

    if (this.envConfigService.hmtPriceSource.includes('coingecko')) {
      if (
        !data ||
        !data[this.envConfigService.hmtPriceFromKey] ||
        !data[this.envConfigService.hmtPriceFromKey][
          this.envConfigService.hmtPriceToKey
        ]
      ) {
        throw new Error('Failed to fetch HMT price from CoinGecko API');
      }
      hmtPrice = parseFloat(
        data[this.envConfigService.hmtPriceFromKey][
          this.envConfigService.hmtPriceToKey
        ],
      );
    } else if (this.envConfigService.hmtPriceSource.includes('coinlore')) {
      if (!data || !data[0] || !data[0].price_usd || data[0].symbol !== 'HMT') {
        throw new Error('Failed to fetch HMT price from Coinlore API');
      }
      hmtPrice = parseFloat(data[0].price_usd);
    } else {
      throw new Error('Unsupported HMT price source');
    }

    await this.cacheManager.set(
      this.redisConfigService.hmtPriceCacheKey,
      hmtPrice,
      this.redisConfigService.cacheHmtPriceTTL,
    );

    return hmtPrice;
  }

  async hCaptchaStats(from: string, to: string): Promise<HcaptchaDailyStats[]> {
    let startDate = dayjs(from);
    const endDate = dayjs(to);
    const dates = [];

    while (startDate <= endDate) {
      dates.push(startDate.format('YYYY-MM-DD'));
      startDate = startDate.add(1, 'day');
    }

    const stats = await Promise.all(
      dates.map(async (date) => {
        const stat: HcaptchaDailyStats = await this.cacheManager.get(
          `${HCAPTCHA_PREFIX}${date}`,
        );
        if (stat) {
          stat.date = date;
        }
        return stat;
      }),
    );

    return stats.filter(Boolean);
  }

  async hCaptchaGeneralStats(): Promise<HcaptchaStats> {
    let startDate = dayjs(HCAPTCHA_STATS_START_DATE);
    const currentDate = dayjs();
    const dates = [];

    while (startDate <= currentDate) {
      dates.push(startDate.format('YYYY-MM'));
      startDate = startDate.add(1, 'month');
    }

    const stats = await Promise.all(
      dates.map(async (date) => {
        const stat: HcaptchaStats = await this.cacheManager.get<HcaptchaStats>(
          `${HCAPTCHA_PREFIX}${date}`,
        );
        return stat;
      }),
    );

    const aggregatedStats: HcaptchaStats = stats.reduce(
      (acc, stat) => {
        if (stat) {
          acc.solved += stat.solved;
        }
        return acc;
      },
      { solved: 0 },
    );

    return aggregatedStats;
  }

  async hmtGeneralStats(): Promise<HmtGeneralStatsDto> {
    const data = await this.cacheManager.get<HmtGeneralStatsDto>(
      this.redisConfigService.hmtGeneralStatsCacheKey,
    );

    return data;
  }

  async hmtDailyStats(from: string, to: string): Promise<HmtDailyStatsData[]> {
    let startDate = dayjs(from);
    const endDate = dayjs(to);
    const dates = [];

    while (startDate <= endDate) {
      dates.push(startDate.format('YYYY-MM-DD'));
      startDate = startDate.add(1, 'day');
    }

    const stats = await Promise.all(
      dates.map(async (date) => {
        const stat: HmtDailyStatsData = await this.cacheManager.get(
          `${HMT_PREFIX}${date}`,
        );
        if (stat) {
          stat.date = date;
        }
        return stat;
      }),
    );

    return stats.filter(Boolean);
  }
}