diff --git a/.gitignore b/.gitignore index 25ff9bd7..c2b206d1 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,4 @@ dmypy.json [._]s[a-rt-v][a-z] [._]ss[a-gi-z] [._]sw[a-p] +*.db diff --git a/setup.cfg b/setup.cfg index c95cf4d7..929f11db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ install_requires = kubernetes openshift coldfront >= 1.1.0 + django-model-utils python-cinderclient python-keystoneclient python-novaclient diff --git a/src/coldfront_plugin_cloud/apps.py b/src/coldfront_plugin_cloud/apps.py index 78c58972..645bacd7 100644 --- a/src/coldfront_plugin_cloud/apps.py +++ b/src/coldfront_plugin_cloud/apps.py @@ -6,3 +6,4 @@ class OpenStackConfig(AppConfig): def ready(self): import coldfront_plugin_cloud.signals # noqa: F401 + import coldfront_plugin_cloud.models.daily_billable_usage # noqa: F401 diff --git a/src/coldfront_plugin_cloud/daily_billable_usage.py b/src/coldfront_plugin_cloud/daily_billable_usage.py new file mode 100644 index 00000000..7bfc5c15 --- /dev/null +++ b/src/coldfront_plugin_cloud/daily_billable_usage.py @@ -0,0 +1,142 @@ +from collections.abc import Iterable + +from coldfront.core.allocation.models import Allocation +from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, +) +from coldfront_plugin_cloud.models.usage_models import UsageInfo, validate_date_str + + +def _rows_to_usage_info(rows: Iterable[AllocationDailyBillableUsage]) -> UsageInfo: + """Build a UsageInfo from ORM rows (one row per SU type). + + Args: + rows: AllocationDailyBillableUsage instances for a single allocation + and date (or any set of rows to collapse into one dict). + + Returns: + UsageInfo mapping SU type names to charge values. Empty when rows + is empty. + + Raises: + TypeError: If rows is None. + ValueError: If a row has an empty su_type. + + Example: + >>> rows = [ + ... AllocationDailyBillableUsage(su_type="OpenStack CPU", value=100), + ... AllocationDailyBillableUsage(su_type="Storage", value=30.12), + ... ] + >>> info = _rows_to_usage_info(rows) + >>> info.root["OpenStack CPU"] + Decimal('100') + >>> info.total_charges + Decimal('130.12') + """ + if rows is None: + raise TypeError("rows must not be None") + + usage_info = UsageInfo({}) + for row in rows: + if not isinstance(row, AllocationDailyBillableUsage): + raise TypeError( + f"each row must be AllocationDailyBillableUsage, got {type(row).__name__}" + ) + if not row.su_type: + raise ValueError(f"usage row id={row.pk} has empty su_type") + usage_info.root[row.su_type] = row.value + return usage_info + + +def get_daily_billable_usage(allocation: Allocation, date: str) -> UsageInfo: + """Load billable usage for one allocation on one day. + + Args: + allocation: ColdFront allocation whose daily_usage_records to read. + date: Calendar day in ``YYYY-MM-DD`` format. + + Returns: + UsageInfo for that allocation and date. Empty dict when no rows exist + (no usage recorded yet for that day). + + Raises: + TypeError: If allocation or date has the wrong type. + ValueError: If allocation is unsaved, date is invalid, or empty. + + Example: + >>> from coldfront_plugin_cloud.daily_billable_usage import get_daily_billable_usage + >>> usage = get_daily_billable_usage(allocation, "2025-11-15") + >>> usage.root.get("OpenStack CPU") + Decimal('100.00') + >>> usage.total_charges # sum of all SU types that day + Decimal('180.12') + """ + if not isinstance(allocation, Allocation): + raise TypeError( + f"allocation must be Allocation, got {type(allocation).__name__}" + ) + if allocation.pk is None: + raise ValueError("allocation must be saved (have a primary key)") + if not isinstance(date, str) or not date.strip(): + raise ValueError("date must be a non-empty YYYY-MM-DD string") + date = validate_date_str(date) + + rows = AllocationDailyBillableUsage.objects.filter(allocation=allocation, date=date) + return _rows_to_usage_info(rows) + + +def get_daily_billable_usage_range( + allocation: Allocation, start_date: str, end_date: str +) -> UsageInfo: + """Load billable usage rows for an allocation across an inclusive date range. + + Args: + allocation: ColdFront allocation to read. + start_date: First day (inclusive), ``YYYY-MM-DD``. + end_date: Last day (inclusive), ``YYYY-MM-DD``. + + Returns: + A single UsageInfo built from all matching rows. Each SU type appears + at most once; if the same SU type exists on multiple days, the last + row processed wins (queryset has no explicit ordering). + + Raises: + TypeError: If allocation or either date has the wrong type. + ValueError: If allocation is unsaved, a date is invalid, or + start_date is after end_date. + + Note: + For per-day breakdowns, call get_daily_billable_usage once per date + or add a separate helper that returns a date-keyed structure + (e.g. CumulativeChargesDict). + + Example: + >>> usage = get_daily_billable_usage_range( + ... allocation, "2025-11-01", "2025-11-30" + ... ) + >>> usage.root # merged across the whole range, not per-day + {'OpenStack CPU': Decimal('110.00'), 'Storage': Decimal('35.00')} + """ + if not isinstance(allocation, Allocation): + raise TypeError( + f"allocation must be Allocation, got {type(allocation).__name__}" + ) + if allocation.pk is None: + raise ValueError("allocation must be saved (have a primary key)") + if not isinstance(start_date, str) or not start_date.strip(): + raise ValueError("start_date must be a non-empty YYYY-MM-DD string") + if not isinstance(end_date, str) or not end_date.strip(): + raise ValueError("end_date must be a non-empty YYYY-MM-DD string") + start_date = validate_date_str(start_date) + end_date = validate_date_str(end_date) + if start_date > end_date: + raise ValueError( + f"start_date {start_date} must be on or before end_date {end_date}" + ) + + rows = AllocationDailyBillableUsage.objects.filter( + allocation=allocation, + date__gte=start_date, + date__lte=end_date, + ) + return _rows_to_usage_info(rows) diff --git a/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py b/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py index eac3e19a..b7d48b77 100644 --- a/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py +++ b/src/coldfront_plugin_cloud/management/commands/fetch_daily_billable_usage.py @@ -12,6 +12,9 @@ from coldfront_plugin_cloud.models import usage_models from coldfront_plugin_cloud.models.usage_models import UsageInfo, validate_date_str from coldfront_plugin_cloud import utils +from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, +) import boto3 from django.core.management.base import BaseCommand @@ -85,10 +88,20 @@ def add_arguments(self, parser): parser.add_argument( "--date", type=str, default=self.previous_day_string, help="Date." ) + parser.add_argument( + "--remove", + action="store_true", + help="Remove usage entries for the specified date instead of fetching.", + ) def handle(self, *args, **options): date = options["date"] validate_date_str(date) + remove = options.get("remove", False) + + if remove: + self.handle_remove(date) + return allocations = self.get_allocations_for_daily_billing() @@ -122,6 +135,8 @@ def handle(self, *args, **options): ) continue + self.store_usage_in_database(allocation, date, new_usage) + # Only update the latest value if the processed date is newer or same date. if not previous_total or date >= previous_total.date: new_total = TotalByDate(date, new_usage.total_charges) @@ -302,3 +317,32 @@ def send_alert_email(cls, allocation: Allocation, resource: Resource, alert_valu if x != allocation.project.pi.email ], ) + + @staticmethod + def store_usage_in_database(allocation: Allocation, date: str, usage_info): + """Store usage information in the database for each SU type. + + Args: + allocation: The allocation to store usage for + date: The date string in YYYY-MM-DD format + usage_info: UsageInfo pydantic model instance with SU type charges + """ + for su_type, value in usage_info.root.items(): + AllocationDailyBillableUsage.objects.update_or_create( + allocation=allocation, + date=date, + su_type=su_type, + defaults={"value": value}, + ) + + @staticmethod + def handle_remove(date: str): + """Remove all usage entries for the specified date. + + Args: + date: The date string in YYYY-MM-DD format for which to remove entries + """ + deleted_count, _ = AllocationDailyBillableUsage.objects.filter( + date=date + ).delete() + logger.info(f"Removed {deleted_count} usage entries for date {date}") diff --git a/src/coldfront_plugin_cloud/management/commands/seed_daily_billable_usage.py b/src/coldfront_plugin_cloud/management/commands/seed_daily_billable_usage.py new file mode 100644 index 00000000..38e3d348 --- /dev/null +++ b/src/coldfront_plugin_cloud/management/commands/seed_daily_billable_usage.py @@ -0,0 +1,153 @@ +import calendar +from datetime import date + +from django.core.management.base import BaseCommand, CommandError + +from coldfront.core.allocation.models import Allocation +from coldfront_plugin_cloud.models import usage_models +from coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage import ( + Command as FetchCommand, +) + +DEFAULT_USAGE = { + "OpenStack CPU": "100.00", + "OpenStack V100 GPU": "50.00", + "Storage": "30.12", +} + + +def dates_in_month(month: str) -> list[str]: + """Return YYYY-MM-DD strings for every day in month (YYYY-MM).""" + usage_models.validate_date_str(f"{month}-01") + year, mon = map(int, month.split("-", 1)) + _, last_day = calendar.monthrange(year, mon) + return [f"{year}-{mon:02d}-{day:02d}" for day in range(1, last_day + 1)] + + +_RAMP_STEP = { + "OpenStack CPU": 5, + "OpenStack V100 GPU": 2, + "Storage": 0.5, +} + + +def usage_for_day( + base: dict[str, str], day_index: int, ramp: bool +) -> usage_models.UsageInfo: + if not ramp: + return usage_models.UsageInfo(base) + ramped = {} + for su_type, amount in base.items(): + step = _RAMP_STEP.get(su_type, 0) + ramped[su_type] = str(float(amount) + day_index * step) + return usage_models.UsageInfo(ramped) + + +def current_month() -> str: + today = date.today() + return f"{today.year}-{today.month:02d}" + + +class Command(BaseCommand): + help = ( + "Insert test AllocationDailyBillableUsage rows for local development. " + "Run from a ColdFront checkout with NERC local_settings (Option B)." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--allocation-id", + type=int, + required=True, + help="ColdFront allocation primary key", + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--date", + help="Single day to seed (YYYY-MM-DD)", + ) + group.add_argument( + "--month", + help="Seed every day in this month (YYYY-MM)", + ) + group.add_argument( + "--current-month", + action="store_true", + help=( + "Seed every day in the current calendar month with 3 default SU types " + "(OpenStack CPU, OpenStack V100 GPU, Storage)" + ), + ) + parser.add_argument( + "--su", + action="append", + metavar="NAME=AMOUNT", + help="SU type and charge; repeat to override the 3 defaults", + ) + parser.add_argument( + "--ramp", + action="store_true", + help="Increase each SU type slightly per day (chart-friendly)", + ) + parser.add_argument( + "--through-today", + action="store_true", + help="With --current-month, only seed days 1 through today (not future days)", + ) + + def handle(self, *args, **options): + try: + allocation = Allocation.objects.get(pk=options["allocation_id"]) + except Allocation.DoesNotExist as exc: + raise CommandError( + f"allocation id={options['allocation_id']} not found" + ) from exc + + base_usage = self._parse_usage(options["su"]) + if options.get("date"): + dates = [usage_models.validate_date_str(options["date"])] + elif options.get("current_month"): + month = current_month() + dates = dates_in_month(month) + if options["through_today"]: + today = date.today().isoformat() + dates = [d for d in dates if d <= today] + else: + dates = dates_in_month(options["month"]) + + total_rows = 0 + for day_index, day in enumerate(dates): + usage_info = usage_for_day(base_usage, day_index, options["ramp"]) + FetchCommand.store_usage_in_database(allocation, day, usage_info) + total_rows += len(usage_info.root) + + su_count = len(base_usage) + self.stdout.write( + self.style.SUCCESS( + f"Seeded {su_count} SU type(s) × {len(dates)} day(s) " + f"= {total_rows} row(s) for allocation {allocation.id}" + ) + ) + if len(dates) > 1: + self.stdout.write(f" dates: {dates[0]} … {dates[-1]}") + for su_type in base_usage: + self.stdout.write(f" {su_type}") + if len(dates) == 1: + for su_type, value in usage_for_day( + base_usage, 0, options["ramp"] + ).root.items(): + self.stdout.write(f" {dates[0]} {su_type}: {value}") + + @staticmethod + def _parse_usage(su_args: list[str] | None) -> dict[str, str]: + if not su_args: + return dict(DEFAULT_USAGE) + usage = {} + for item in su_args: + if "=" not in item: + raise CommandError( + f"expected NAME=AMOUNT, got {item!r} (e.g. 'OpenStack CPU=100.00')" + ) + name, amount = item.split("=", 1) + usage[name.strip()] = amount.strip() + return usage diff --git a/src/coldfront_plugin_cloud/migrations/0001_initial.py b/src/coldfront_plugin_cloud/migrations/0001_initial.py new file mode 100644 index 00000000..abdd5d21 --- /dev/null +++ b/src/coldfront_plugin_cloud/migrations/0001_initial.py @@ -0,0 +1,95 @@ +# Generated manually for coldfront_plugin_cloud + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + ("allocation", "__first__"), + ] + + operations = [ + migrations.CreateModel( + name="AllocationDailyBillableUsage", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="modified", + ), + ), + ( + "date", + models.DateField( + help_text="The date for which this usage was recorded" + ), + ), + ( + "su_type", + models.CharField( + help_text="The type of Service Unit (e.g., OpenStack CPU, OpenStack V100 GPU)", + max_length=255, + ), + ), + ( + "value", + models.DecimalField( + decimal_places=2, + help_text="The usage value/cost for this SU type on this date", + max_digits=12, + ), + ), + ( + "allocation", + models.ForeignKey( + help_text="The allocation this usage belongs to", + on_delete=django.db.models.deletion.CASCADE, + related_name="daily_usage_records", + to="allocation.allocation", + ), + ), + ], + options={ + "db_table": "coldfront_plugin_cloud_allocationdailybillableusage", + "ordering": ["-date", "allocation", "su_type"], + }, + ), + migrations.AddIndex( + model_name="allocationdailybillableusage", + index=models.Index( + fields=["allocation", "date"], name="coldfront_p_allocat_5c8e3d_idx" + ), + ), + migrations.AddIndex( + model_name="allocationdailybillableusage", + index=models.Index(fields=["date"], name="coldfront_p_date_3e8a9e_idx"), + ), + migrations.AlterUniqueTogether( + name="allocationdailybillableusage", + unique_together={("allocation", "date", "su_type")}, + ), + ] diff --git a/src/coldfront_plugin_cloud/migrations/__init__.py b/src/coldfront_plugin_cloud/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/coldfront_plugin_cloud/models/daily_billable_usage.py b/src/coldfront_plugin_cloud/models/daily_billable_usage.py new file mode 100644 index 00000000..b71c7b47 --- /dev/null +++ b/src/coldfront_plugin_cloud/models/daily_billable_usage.py @@ -0,0 +1,36 @@ +from django.db import models +from model_utils.models import TimeStampedModel +from coldfront.core.allocation.models import Allocation + + +class AllocationDailyBillableUsage(TimeStampedModel): + """Stores daily billable usage for allocations by SU type.""" + + allocation = models.ForeignKey( + Allocation, + on_delete=models.CASCADE, + related_name="daily_usage_records", + help_text="The allocation this usage belongs to", + ) + date = models.DateField(help_text="The date for which this usage was recorded") + su_type = models.CharField( + max_length=255, + help_text="The type of Service Unit (e.g., OpenStack CPU, OpenStack V100 GPU)", + ) + value = models.DecimalField( + max_digits=12, + decimal_places=2, + help_text="The usage value/cost for this SU type on this date", + ) + + class Meta: + db_table = "coldfront_plugin_cloud_allocationdailybillableusage" + unique_together = [["allocation", "date", "su_type"]] + indexes = [ + models.Index(fields=["allocation", "date"]), + models.Index(fields=["date"]), + ] + ordering = ["-date", "allocation", "su_type"] + + def __str__(self): + return f"{self.allocation.id} - {self.date} - {self.su_type}: {self.value}" diff --git a/src/coldfront_plugin_cloud/tests/unit/test_daily_billable_usage.py b/src/coldfront_plugin_cloud/tests/unit/test_daily_billable_usage.py new file mode 100644 index 00000000..f3d44d93 --- /dev/null +++ b/src/coldfront_plugin_cloud/tests/unit/test_daily_billable_usage.py @@ -0,0 +1,350 @@ +import time +from decimal import Decimal + +from django.core.exceptions import ValidationError +from django.core.management import call_command +from django.db import IntegrityError + +from coldfront.core.allocation.models import Allocation +from coldfront_plugin_cloud.daily_billable_usage import ( + _rows_to_usage_info, + get_daily_billable_usage, + get_daily_billable_usage_range, +) +from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, +) +from coldfront_plugin_cloud.tests import base + + +class TestRowsToUsageInfo(base.TestBase): + def _new_allocation(self): + resource = self.new_openstack_resource() + project = self.new_project() + return self.new_allocation(project=project, resource=resource, quantity=1) + + def test_empty_iterable(self): + # Verify the helper returns a valid empty UsageInfo for "no rows" cases. + usage = _rows_to_usage_info([]) + self.assertEqual(usage.root, {}) + self.assertEqual(usage.total_charges, Decimal("0")) + + def test_multiple_rows(self): + # Verify multiple ORM rows collapse into a single SU->Decimal mapping with a correct total. + rows = [ + AllocationDailyBillableUsage(su_type="OpenStack CPU", value=Decimal("100")), + AllocationDailyBillableUsage(su_type="Storage", value=Decimal("30.12")), + ] + usage = _rows_to_usage_info(rows) + self.assertEqual(usage.root["OpenStack CPU"], Decimal("100")) + self.assertEqual(usage.root["Storage"], Decimal("30.12")) + self.assertEqual(usage.total_charges, Decimal("130.12")) + + def test_rows_is_none(self): + # Defensive programming: None is a bug at the call site and should fail loudly. + with self.assertRaises(TypeError): + _rows_to_usage_info(None) + + def test_non_model_row(self): + # Ensure we don't silently accept unexpected row objects (helps catch query/fixture mistakes). + with self.assertRaises(TypeError) as ctx: + _rows_to_usage_info(["not-a-row"]) + self.assertIn("AllocationDailyBillableUsage", str(ctx.exception)) + + def test_empty_su_type(self): + # Enforce that every row has a usable key; empty SU type would corrupt the usage dict. + allocation = self._new_allocation() + row = AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date="2025-11-15", + su_type="", + value=Decimal("1.00"), + ) + with self.assertRaises(ValueError) as ctx: + _rows_to_usage_info([row]) + self.assertIn(f"id={row.pk}", str(ctx.exception)) + + +class TestGetDailyBillableUsage(base.TestBase): + def _new_allocation(self): + resource = self.new_openstack_resource() + project = self.new_project() + return self.new_allocation(project=project, resource=resource, quantity=1) + + def _create_usage_row(self, allocation, date, su_type, value): + return AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date=date, + su_type=su_type, + value=Decimal(value), + ) + + def test_happy_path_via_seed_command(self): + # Integration sanity check: seed rows via command, then read them back through the API. + allocation = self._new_allocation() + call_command( + "seed_daily_billable_usage", + allocation_id=allocation.id, + date="2025-11-15", + ) + + usage = get_daily_billable_usage(allocation, "2025-11-15") + + self.assertEqual(usage.root["OpenStack CPU"], Decimal("100.00")) + self.assertEqual(usage.root["OpenStack V100 GPU"], Decimal("50.00")) + self.assertEqual(usage.root["Storage"], Decimal("30.12")) + self.assertEqual(usage.total_charges, Decimal("180.12")) + + def test_no_rows_returns_empty_usage_info(self): + # A missing day should return an empty UsageInfo rather than raising or returning None. + allocation = self._new_allocation() + usage = get_daily_billable_usage(allocation, "2025-11-15") + self.assertEqual(usage.root, {}) + self.assertEqual(usage.total_charges, Decimal("0")) + + def test_wrong_allocation_type(self): + # Guardrails: callers passing the wrong object type get a clear, early error. + with self.assertRaises(TypeError) as ctx: + get_daily_billable_usage("not-an-allocation", "2025-11-15") + self.assertIn("Allocation", str(ctx.exception)) + + def test_unsaved_allocation(self): + # Unsaved allocations can't be queried reliably, so we reject them explicitly. + allocation = Allocation() + with self.assertRaises(ValueError) as ctx: + get_daily_billable_usage(allocation, "2025-11-15") + self.assertIn("primary key", str(ctx.exception)) + + def test_empty_date(self): + # Empty date strings are ambiguous and should be rejected before hitting the ORM. + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage(allocation, "") + + def test_whitespace_date(self): + # Whitespace-only dates are treated as empty input and rejected. + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage(allocation, " ") + + def test_invalid_date(self): + # Invalid calendar dates should fail validation rather than executing a query. + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage(allocation, "2025-13-01") + + def test_excludes_other_allocation_and_date(self): + # Verify the query is correctly scoped by allocation AND date (no accidental cross-talk). + allocation = self._new_allocation() + other_allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-15", "OpenStack CPU", "100.00") + self._create_usage_row(allocation, "2025-11-16", "OpenStack CPU", "200.00") + self._create_usage_row( + other_allocation, "2025-11-15", "OpenStack CPU", "999.00" + ) + + usage = get_daily_billable_usage(allocation, "2025-11-15") + + self.assertEqual(usage.root, {"OpenStack CPU": Decimal("100.00")}) + self.assertEqual(usage.total_charges, Decimal("100.00")) + + +class TestGetDailyBillableUsageRange(base.TestBase): + def _new_allocation(self): + resource = self.new_openstack_resource() + project = self.new_project() + return self.new_allocation(project=project, resource=resource, quantity=1) + + def _create_usage_row(self, allocation, date, su_type, value): + return AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date=date, + su_type=su_type, + value=Decimal(value), + ) + + def test_inclusive_range(self): + # Range queries should include both endpoints and exclude rows outside the window. + allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-01", "OpenStack CPU", "10.00") + self._create_usage_row(allocation, "2025-11-15", "Storage", "20.00") + self._create_usage_row(allocation, "2025-11-30", "OpenStack V100 GPU", "30.00") + self._create_usage_row(allocation, "2025-12-01", "OpenStack CPU", "99.00") + + usage = get_daily_billable_usage_range(allocation, "2025-11-01", "2025-11-30") + + self.assertEqual(usage.root["OpenStack CPU"], Decimal("10.00")) + self.assertEqual(usage.root["Storage"], Decimal("20.00")) + self.assertEqual(usage.root["OpenStack V100 GPU"], Decimal("30.00")) + self.assertEqual(usage.total_charges, Decimal("60.00")) + + def test_empty_when_no_matching_rows(self): + # A valid range with no data should return an empty UsageInfo, not an error. + allocation = self._new_allocation() + usage = get_daily_billable_usage_range(allocation, "2025-11-01", "2025-11-30") + self.assertEqual(usage.root, {}) + self.assertEqual(usage.total_charges, Decimal("0")) + + def test_start_date_after_end_date(self): + # Detect inverted ranges early to avoid confusing empty results. + allocation = self._new_allocation() + with self.assertRaises(ValueError) as ctx: + get_daily_billable_usage_range(allocation, "2025-11-30", "2025-11-01") + self.assertIn("start_date", str(ctx.exception)) + self.assertIn("end_date", str(ctx.exception)) + + def test_duplicate_su_type_last_row_wins(self): + # Document current merge semantics: duplicate SU types collapse to one value based on iteration order. + allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-01", "OpenStack CPU", "100.00") + self._create_usage_row(allocation, "2025-11-15", "OpenStack CPU", "200.00") + + usage = get_daily_billable_usage_range(allocation, "2025-11-01", "2025-11-15") + + self.assertEqual(usage.root["OpenStack CPU"], Decimal("100.00")) + + def test_unsaved_allocation(self): + # Same guardrail as single-day reads: unsaved allocations can't be queried. + allocation = Allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage_range(allocation, "2025-11-01", "2025-11-30") + + def test_invalid_start_date(self): + # Validate both endpoints before querying so bad input fails consistently. + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage_range(allocation, "2025-13-01", "2025-11-30") + + def test_invalid_end_date(self): + # Validate both endpoints before querying so bad input fails consistently. + allocation = self._new_allocation() + with self.assertRaises(ValueError): + get_daily_billable_usage_range(allocation, "2025-11-01", "not-a-date") + + +class TestAllocationDailyBillableUsageModel(base.TestBase): + def _new_allocation(self): + resource = self.new_openstack_resource() + project = self.new_project() + return self.new_allocation(project=project, resource=resource, quantity=1) + + def _create_usage_row(self, allocation, date, su_type, value): + return AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date=date, + su_type=su_type, + value=Decimal(value), + ) + + def test_str(self): + # A stable __str__ helps debugging/logging/admin views when inspecting usage rows. + allocation = self._new_allocation() + row = self._create_usage_row( + allocation, "2025-11-15", "OpenStack CPU", "100.00" + ) + self.assertEqual( + str(row), + f"{allocation.id} - 2025-11-15 - OpenStack CPU: 100.00", + ) + + def test_unique_together_raises_on_duplicate(self): + # Enforce one row per (allocation, date, su_type) so upserts don't create duplicates. + allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-15", "OpenStack CPU", "100.00") + with self.assertRaises(IntegrityError): + AllocationDailyBillableUsage.objects.create( + allocation=allocation, + date="2025-11-15", + su_type="OpenStack CPU", + value=Decimal("200.00"), + ) + + def test_allocation_delete_cascades_to_usage_rows(self): + # Foreign key cascade prevents orphaned usage rows when an allocation is deleted. + allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-15", "OpenStack CPU", "100.00") + self._create_usage_row(allocation, "2025-11-16", "Storage", "30.00") + self.assertEqual(AllocationDailyBillableUsage.objects.count(), 2) + + allocation.delete() + + self.assertEqual(AllocationDailyBillableUsage.objects.count(), 0) + + def test_meta_ordering(self): + # Ordering affects deterministic reads/merges; this locks the contract to the model definition. + self.assertEqual( + AllocationDailyBillableUsage._meta.ordering, + ["-date", "allocation", "su_type"], + ) + allocation_a = self._new_allocation() + allocation_b = self._new_allocation() + self._create_usage_row(allocation_a, "2025-11-01", "Storage", "10.00") + self._create_usage_row(allocation_a, "2025-11-15", "OpenStack CPU", "20.00") + self._create_usage_row(allocation_b, "2025-11-15", "OpenStack CPU", "30.00") + + rows = list(AllocationDailyBillableUsage.objects.all()) + self.assertEqual(len(rows), 3) + self.assertEqual(rows[0].date.isoformat(), "2025-11-15") + self.assertEqual(rows[1].date.isoformat(), "2025-11-15") + self.assertEqual(rows[2].date.isoformat(), "2025-11-01") + # Same date: ordering uses Allocation model default ordering via FK. + nov_15_allocation_ids = {rows[0].allocation_id, rows[1].allocation_id} + self.assertEqual(nov_15_allocation_ids, {allocation_a.id, allocation_b.id}) + + def test_value_stores_two_decimal_places(self): + # Value is money-like and must round-trip without float precision loss. + allocation = self._new_allocation() + row = self._create_usage_row(allocation, "2025-11-15", "Storage", "30.12") + row.refresh_from_db() + self.assertEqual(row.value, Decimal("30.12")) + + def test_value_rejects_overflow_beyond_max_digits(self): + # Schema protection: reject invoice values that exceed the declared DecimalField size. + allocation = self._new_allocation() + row = AllocationDailyBillableUsage( + allocation=allocation, + date="2025-11-15", + su_type="OpenStack CPU", + value=Decimal("10000000000.00"), + ) + with self.assertRaises(ValidationError): + row.full_clean() + + def test_timestamps_set_on_create_and_modified_on_update(self): + # TimeStampedModel should set created/modified and advance modified on updates. + allocation = self._new_allocation() + row = self._create_usage_row( + allocation, "2025-11-15", "OpenStack CPU", "100.00" + ) + created_at = row.created + modified_at = row.modified + self.assertIsNotNone(created_at) + self.assertIsNotNone(modified_at) + + time.sleep(0.02) + row.value = Decimal("150.00") + row.save() + row.refresh_from_db() + + self.assertEqual(row.created, created_at) + self.assertGreater(row.modified, modified_at) + + def test_related_name_daily_usage_records(self): + # Reverse relation is the primary way callers will traverse allocation -> daily usage rows. + allocation = self._new_allocation() + other_allocation = self._new_allocation() + self._create_usage_row(allocation, "2025-11-15", "OpenStack CPU", "100.00") + self._create_usage_row(allocation, "2025-11-16", "Storage", "30.00") + self._create_usage_row( + other_allocation, "2025-11-15", "OpenStack CPU", "999.00" + ) + + self.assertEqual(allocation.daily_usage_records.count(), 2) + self.assertEqual( + allocation.daily_usage_records.filter(date="2025-11-15").count(), + 1, + ) + self.assertEqual( + allocation.daily_usage_records.get(date="2025-11-15").value, + Decimal("100.00"), + ) diff --git a/src/coldfront_plugin_cloud/tests/unit/test_fetch_daily_billable_usage.py b/src/coldfront_plugin_cloud/tests/unit/test_fetch_daily_billable_usage.py index 477801ff..04c73f8d 100644 --- a/src/coldfront_plugin_cloud/tests/unit/test_fetch_daily_billable_usage.py +++ b/src/coldfront_plugin_cloud/tests/unit/test_fetch_daily_billable_usage.py @@ -1,4 +1,5 @@ import io +from decimal import Decimal from unittest import mock from unittest.mock import Mock, patch @@ -240,3 +241,197 @@ def test_send_alert_email(self): receiver_list=[allocation_1.project.pi.email], cc=[manager.email], ) + + @patch( + "coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage.RESOURCES_DAILY_ENABLED", + ["FakeProd"], + ) + @patch( + "coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage.Command.get_allocation_usage" + ) + def test_database_insertion_and_removal(self, mock_get_allocation_usage): + """Test database insertion, updates, and removal of usage entries.""" + from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, + ) + + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo( + {"OpenStack CPU": "100.00", "OpenStack GPU": "50.00"} + ), + usage_models.UsageInfo({"Storage": "30.12"}), + ] + + fakeprod = self.new_openstack_resource( + name="FakeProd", internal_name="FakeProd" + ) + prod_project = self.new_project() + allocation_1 = self.new_allocation( + project=prod_project, resource=fakeprod, quantity=1, status="Active" + ) + utils.set_attribute_on_allocation( + allocation_1, attributes.ALLOCATION_PROJECT_ID, "test-allocation-1" + ) + + # Verify no entries before running command + self.assertEqual(AllocationDailyBillableUsage.objects.count(), 0) + + # Test initial insertion + call_command("fetch_daily_billable_usage", date="2025-11-15") + + # Verify database entries were created + usage_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_1, date="2025-11-15" + ) + self.assertEqual(usage_entries.count(), 3) + + # Check individual SU types + cpu_usage = usage_entries.get(su_type="OpenStack CPU") + self.assertEqual(cpu_usage.value, Decimal("100.00")) + + gpu_usage = usage_entries.get(su_type="OpenStack GPU") + self.assertEqual(gpu_usage.value, Decimal("50.00")) + + storage_usage = usage_entries.get(su_type="Storage") + self.assertEqual(storage_usage.value, Decimal("30.12")) + + # Test update_or_create by running again with different values for same date + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo( + {"OpenStack CPU": "110.00", "OpenStack GPU": "55.00"} + ), + usage_models.UsageInfo({"Storage": "35.00"}), + ] + call_command("fetch_daily_billable_usage", date="2025-11-15") + + # Should still have 3 entries (not duplicates) + usage_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_1, date="2025-11-15" + ) + self.assertEqual(usage_entries.count(), 3) + + # Check updated values + cpu_usage = usage_entries.get(su_type="OpenStack CPU") + self.assertEqual(cpu_usage.value, Decimal("110.00")) + + # Add data for another date to test selective removal + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo({"OpenStack CPU": "120.00"}), + usage_models.UsageInfo({"Storage": "40.00"}), + ] + call_command("fetch_daily_billable_usage", date="2025-11-16") + + # Verify data exists for both dates + self.assertEqual( + AllocationDailyBillableUsage.objects.filter(date="2025-11-15").count(), 3 + ) + self.assertEqual( + AllocationDailyBillableUsage.objects.filter(date="2025-11-16").count(), 2 + ) + + # Test removal - remove data for 2025-11-15 + call_command("fetch_daily_billable_usage", date="2025-11-15", remove=True) + + # Verify data for 2025-11-15 is deleted + self.assertEqual( + AllocationDailyBillableUsage.objects.filter(date="2025-11-15").count(), 0 + ) + + # Verify data for 2025-11-16 still exists + self.assertEqual( + AllocationDailyBillableUsage.objects.filter(date="2025-11-16").count(), 2 + ) + + @patch( + "coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage.RESOURCES_DAILY_ENABLED", + ["FakeProd"], + ) + @patch( + "coldfront_plugin_cloud.management.commands.fetch_daily_billable_usage.Command.get_allocation_usage" + ) + def test_multiple_allocations_same_date(self, mock_get_allocation_usage): + """Test that multiple allocations can store usage for the same date.""" + from coldfront_plugin_cloud.models.daily_billable_usage import ( + AllocationDailyBillableUsage, + ) + + fakeprod = self.new_openstack_resource( + name="FakeProd", internal_name="FakeProd" + ) + prod_project1 = self.new_project() + prod_project2 = self.new_project() + + allocation_1 = self.new_allocation( + project=prod_project1, resource=fakeprod, quantity=1, status="Active" + ) + allocation_2 = self.new_allocation( + project=prod_project2, resource=fakeprod, quantity=1, status="Active" + ) + + utils.set_attribute_on_allocation( + allocation_1, attributes.ALLOCATION_PROJECT_ID, "test-allocation-1" + ) + utils.set_attribute_on_allocation( + allocation_2, attributes.ALLOCATION_PROJECT_ID, "test-allocation-2" + ) + + # Mock returns for both allocations + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo({"OpenStack CPU": "100.00"}), + usage_models.UsageInfo({"Storage": "30.00"}), + usage_models.UsageInfo({"OpenStack CPU": "200.00"}), + usage_models.UsageInfo({"Storage": "50.00"}), + ] + + call_command("fetch_daily_billable_usage", date="2025-11-15") + + # Verify both allocations have data + alloc1_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_1, date="2025-11-15" + ) + alloc2_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_2, date="2025-11-15" + ) + + self.assertEqual(alloc1_entries.count(), 2) + self.assertEqual(alloc2_entries.count(), 2) + + # Verify values are correct for each allocation + self.assertEqual( + alloc1_entries.get(su_type="OpenStack CPU").value, Decimal("100.00") + ) + self.assertEqual( + alloc2_entries.get(su_type="OpenStack CPU").value, Decimal("200.00") + ) + + # Test updating with same date but different values - should update, not error + mock_get_allocation_usage.side_effect = [ + usage_models.UsageInfo({"OpenStack CPU": "150.00"}), + usage_models.UsageInfo({"Storage": "35.00"}), + usage_models.UsageInfo({"OpenStack CPU": "250.00"}), + usage_models.UsageInfo({"Storage": "55.00"}), + ] + + # This should not raise any errors and should update existing values + call_command("fetch_daily_billable_usage", date="2025-11-15") + + # Verify counts remain the same (no duplicates) + alloc1_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_1, date="2025-11-15" + ) + alloc2_entries = AllocationDailyBillableUsage.objects.filter( + allocation=allocation_2, date="2025-11-15" + ) + + self.assertEqual(alloc1_entries.count(), 2) + self.assertEqual(alloc2_entries.count(), 2) + + # Verify values were updated + self.assertEqual( + alloc1_entries.get(su_type="OpenStack CPU").value, Decimal("150.00") + ) + self.assertEqual(alloc1_entries.get(su_type="Storage").value, Decimal("35.00")) + self.assertEqual( + alloc2_entries.get(su_type="OpenStack CPU").value, Decimal("250.00") + ) + self.assertEqual(alloc2_entries.get(su_type="Storage").value, Decimal("55.00"))