Skip to content

Commit 679355c

Browse files
Merge pull request #33 from towner-10/compression-attributes
Compression OrderBy & SegmentBy
2 parents 4c7b549 + 2a5d723 commit 679355c

25 files changed

Lines changed: 2857 additions & 14 deletions

samples/Eftdb.Samples.Shared/Models/DeviceReading.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Example.DataAccess.Models
66
{
7-
[Hypertable(nameof(Time), ChunkSkipColumns = new[] { "Time" }, ChunkTimeInterval = "1 day", EnableCompression = true)]
7+
[Hypertable(nameof(Time), ChunkSkipColumns = new[] { "Time" }, ChunkTimeInterval = "1 day", EnableCompression = true, CompressionSegmentBy = new[] { "DeviceId" }, CompressionOrderBy = new[] { "Time DESC" })]
88
[Index(nameof(Time), Name = "ix_device_readings_time")]
99
[PrimaryKey(nameof(Id), nameof(Time))]
1010
[ReorderPolicy("ix_device_readings_time", InitialStart = "2025-09-23T09:15:19.3905112Z", ScheduleInterval = "1 day", MaxRuntime = "00:00:00", RetryPeriod = "00:05:00", MaxRetries = 3)]

src/Eftdb.Design/Scaffolding/HypertableAnnotationApplier.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,18 @@ public void ApplyAnnotations(DatabaseTable table, object featureInfo)
2727
table[HypertableAnnotations.ChunkSkipColumns] = string.Join(",", info.ChunkSkipColumns);
2828
}
2929

30+
// Apply SegmentBy annotation if present
31+
if (info.CompressionSegmentBy.Count > 0)
32+
{
33+
table[HypertableAnnotations.CompressionSegmentBy] = string.Join(", ", info.CompressionSegmentBy);
34+
}
35+
36+
// Apply OrderBy annotation if present
37+
if (info.CompressionOrderBy.Count > 0)
38+
{
39+
table[HypertableAnnotations.CompressionOrderBy] = string.Join(", ", info.CompressionOrderBy);
40+
}
41+
3042
if (info.AdditionalDimensions.Count > 0)
3143
{
3244
table[HypertableAnnotations.AdditionalDimensions] = JsonSerializer.Serialize(info.AdditionalDimensions);

src/Eftdb.Design/Scaffolding/HypertableScaffoldingExtractor.cs

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ public sealed record HypertableInfo(
1313
string TimeColumnName,
1414
string ChunkTimeInterval,
1515
bool CompressionEnabled,
16+
List<string> CompressionSegmentBy,
17+
List<string> CompressionOrderBy,
1618
List<string> ChunkSkipColumns,
1719
List<Dimension> AdditionalDimensions
1820
);
@@ -32,6 +34,7 @@ List<Dimension> AdditionalDimensions
3234

3335
GetHypertableSettings(connection, hypertables, compressionSettings);
3436
GetChunkSkipColumns(connection, hypertables);
37+
GetCompressionConfiguration(connection, hypertables);
3538

3639
// Convert to object dictionary to match interface
3740
return hypertables.ToDictionary(
@@ -99,6 +102,8 @@ FROM timescaledb_information.dimensions
99102
TimeColumnName: columnName,
100103
ChunkTimeInterval: chunkInterval.ToString(),
101104
CompressionEnabled: compressionEnabled,
105+
CompressionSegmentBy: [],
106+
CompressionOrderBy: [],
102107
ChunkSkipColumns: [],
103108
AdditionalDimensions: []
104109
);
@@ -159,5 +164,58 @@ FROM _timescaledb_catalog.chunk_column_stats AS ccs
159164
}
160165
}
161166
}
167+
168+
private static void GetCompressionConfiguration(DbConnection connection, Dictionary<(string, string), HypertableInfo> hypertables)
169+
{
170+
using DbCommand command = connection.CreateCommand();
171+
172+
// This view provides the column-level details for compression.
173+
// segmentby_column_index is not null for segment columns.
174+
// orderby_column_index is not null for order columns.
175+
command.CommandText = @"
176+
SELECT
177+
hypertable_schema,
178+
hypertable_name,
179+
attname,
180+
segmentby_column_index,
181+
orderby_column_index,
182+
orderby_asc,
183+
orderby_nullsfirst
184+
FROM timescaledb_information.compression_settings
185+
ORDER BY hypertable_schema, hypertable_name, segmentby_column_index, orderby_column_index;";
186+
187+
using DbDataReader reader = command.ExecuteReader();
188+
while (reader.Read())
189+
{
190+
string schema = reader.GetString(0);
191+
string name = reader.GetString(1);
192+
string columnName = reader.GetString(2);
193+
194+
// Find the corresponding hypertable info
195+
if (!hypertables.TryGetValue((schema, name), out HypertableInfo? info))
196+
{
197+
continue;
198+
}
199+
200+
// Handle SegmentBy
201+
if (!reader.IsDBNull(3)) // segmentby_column_index
202+
{
203+
info.CompressionSegmentBy.Add(columnName);
204+
}
205+
206+
// Handle OrderBy
207+
if (!reader.IsDBNull(4)) // orderby_column_index
208+
{
209+
bool isAscending = reader.GetBoolean(5);
210+
bool isNullsFirst = reader.GetBoolean(6);
211+
212+
string direction = isAscending ? "ASC" : "DESC";
213+
bool isDefaultNulls = (isAscending && !isNullsFirst) || (!isAscending && isNullsFirst);
214+
string nulls = isDefaultNulls ? "" : (isNullsFirst ? " NULLS FIRST" : " NULLS LAST");
215+
216+
info.CompressionOrderBy.Add($"{columnName} {direction}{nulls}");
217+
}
218+
}
219+
}
162220
}
163221
}

src/Eftdb/Configuration/Hypertable/HypertableAnnotations.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ public static class HypertableAnnotations
88
public const string IsHypertable = "TimescaleDB:IsHypertable";
99
public const string HypertableTimeColumn = "TimescaleDB:TimeColumnName";
1010
public const string EnableCompression = "TimescaleDB:EnableCompression";
11+
public const string CompressionSegmentBy = "TimescaleDB:CompressionSegmentBy";
12+
public const string CompressionOrderBy = "TimescaleDB:CompressionOrderBy";
1113
public const string MigrateData = "TimescaleDB:MigrateData";
1214
public const string ChunkTimeInterval = "TimescaleDB:ChunkTimeInterval";
1315
public const string ChunkSkipColumns = "TimescaleDB:ChunkSkipColumns";

src/Eftdb/Configuration/Hypertable/HypertableAttribute.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,25 @@ public sealed class HypertableAttribute : Attribute
1414
/// </summary>
1515
public bool EnableCompression { get; set; } = false;
1616

17+
/// <summary>
18+
/// Specifies the columns to group by when compressing the hypertable.
19+
/// Maps to <c>timescaledb.compress_segmentby</c>.
20+
/// </summary>
21+
/// <example>
22+
/// <code>[Hypertable("time", CompressionSegmentBy = ["device_id", "tenant_id"])]</code>
23+
/// </example>
24+
public string[]? CompressionSegmentBy { get; set; } = null;
25+
26+
/// <summary>
27+
/// Specifies the columns to order by within each compressed segment.
28+
/// Maps to <c>timescaledb.compress_orderby</c>.
29+
/// Since attributes cannot use Expressions, you must specify the full SQL syntax if direction is needed.
30+
/// </summary>
31+
/// <example>
32+
/// <code>[Hypertable("time", CompressionOrderBy = ["time DESC", "value ASC NULLS LAST"])]</code>
33+
/// </example>
34+
public string[]? CompressionOrderBy { get; set; } = null;
35+
1736
/// <summary>
1837
/// Specifies whether existing data should be migrated when converting a table to a hypertable.
1938
/// </summary>

src/Eftdb/Configuration/Hypertable/HypertableConvention.cs

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,24 @@ public void ProcessEntityTypeAdded(IConventionEntityTypeBuilder entityTypeBuilde
4444

4545
if (attribute.ChunkSkipColumns != null && attribute.ChunkSkipColumns.Length > 0)
4646
{
47-
/// Chunk skipping requires compression to be enabled
47+
// Chunk skipping requires compression to be enabled
4848
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
4949
entityTypeBuilder.HasAnnotation(HypertableAnnotations.ChunkSkipColumns, string.Join(",", attribute.ChunkSkipColumns));
5050
}
51+
52+
if (attribute.CompressionSegmentBy != null && attribute.CompressionSegmentBy.Length > 0)
53+
{
54+
// SegmentBy requires compression to be enabled
55+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
56+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.CompressionSegmentBy, string.Join(", ", attribute.CompressionSegmentBy));
57+
}
58+
59+
if (attribute.CompressionOrderBy != null && attribute.CompressionOrderBy.Length > 0)
60+
{
61+
// OrderBy requires compression to be enabled
62+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
63+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.CompressionOrderBy, string.Join(", ", attribute.CompressionOrderBy));
64+
}
5165
}
5266
}
5367
}

src/Eftdb/Configuration/Hypertable/HypertableTypeBuilder.cs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,62 @@ public static EntityTypeBuilder<TEntity> EnableCompression<TEntity>(
128128
return entityTypeBuilder;
129129
}
130130

131+
/// <summary>
132+
/// Specifies the columns to group by when compressing the hypertable (SegmentBy).
133+
/// </summary>
134+
/// <remarks>
135+
/// Valid settings for <c>timescaledb.compress_segmentby</c>.
136+
/// Columns used for segmenting are not compressed themselves but are used as keys to group rows.
137+
/// Good candidates are columns with low cardinality (e.g., "device_id", "tenant_id").
138+
/// </remarks>
139+
public static EntityTypeBuilder<TEntity> WithCompressionSegmentBy<TEntity>(
140+
this EntityTypeBuilder<TEntity> entityTypeBuilder,
141+
params Expression<Func<TEntity, object>>[] segmentByColumns) where TEntity : class
142+
{
143+
string[] columnNames = [.. segmentByColumns.Select(GetPropertyName)];
144+
145+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.CompressionSegmentBy, string.Join(", ", columnNames));
146+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
147+
148+
return entityTypeBuilder;
149+
}
150+
151+
/// <summary>
152+
/// Specifies the columns to order by within each compressed segment using explicit OrderBy definitions.
153+
/// </summary>
154+
/// <remarks>
155+
/// Uses the <see cref="OrderByBuilder"/> to define direction and null handling.
156+
/// Example: <c>.WithCompressionOrderBy(OrderByBuilder.For&lt;T&gt;(x => x.Time).Descending())</c>
157+
/// </remarks>
158+
public static EntityTypeBuilder<TEntity> WithCompressionOrderBy<TEntity>(
159+
this EntityTypeBuilder<TEntity> entityTypeBuilder,
160+
params OrderBy[] orderByRules) where TEntity : class
161+
{
162+
string annotationValue = string.Join(", ", orderByRules.Select(r => r.ToSql()));
163+
164+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.CompressionOrderBy, annotationValue);
165+
entityTypeBuilder.HasAnnotation(HypertableAnnotations.EnableCompression, true);
166+
167+
return entityTypeBuilder;
168+
}
169+
170+
/// <summary>
171+
/// Specifies the columns to order by within each compressed segment using the OrderBySelector.
172+
/// </summary>
173+
/// <remarks>
174+
/// Provides a simplified syntax for defining order.
175+
/// Example: <c>.WithCompressionOrderBy(s => [s.ByDescending(x => x.Time), s.By(x => x.Value)])</c>
176+
/// </remarks>
177+
public static EntityTypeBuilder<TEntity> WithCompressionOrderBy<TEntity>(
178+
this EntityTypeBuilder<TEntity> entityTypeBuilder,
179+
Func<OrderBySelector<TEntity>, IEnumerable<OrderBy>> orderSelector) where TEntity : class
180+
{
181+
OrderBySelector<TEntity> selector = new();
182+
IEnumerable<OrderBy> rules = orderSelector(selector);
183+
184+
return entityTypeBuilder.WithCompressionOrderBy([.. rules]);
185+
}
186+
131187
/// <summary>
132188
/// Specifies whether existing data should be migrated when converting a table to a hypertable.
133189
/// </summary>
Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
using System.Linq.Expressions;
2+
using System.Text;
3+
4+
namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable
5+
{
6+
/// <summary>
7+
/// Represents an ordering specification for a column.
8+
/// </summary>
9+
/// <param name="columnName">The name of the column to order by.</param>
10+
/// <param name="isAscending">
11+
/// If true, orders Ascending (ASC).
12+
/// If false, orders Descending (DESC).
13+
/// If null, uses database default (ASC).
14+
/// </param>
15+
/// <param name="nullsFirst">
16+
/// If true, forces NULLS FIRST.
17+
/// If false, forces NULLS LAST.
18+
/// If null, uses database default (NULLS LAST for ASC, NULLS FIRST for DESC).
19+
/// </param>
20+
public class OrderBy(string columnName, bool? isAscending = null, bool? nullsFirst = null)
21+
{
22+
/// <summary>The name of the column to order by.</summary>
23+
public string ColumnName { get; } = columnName;
24+
25+
/// <summary>Ordering direction. True for ASC, false for DESC, null for database default.</summary>
26+
public bool? IsAscending { get; } = isAscending;
27+
28+
/// <summary>Null sorting behavior. True for NULLS FIRST, false for NULLS LAST, null for database default.</summary>
29+
public bool? NullsFirst { get; } = nullsFirst;
30+
31+
/// <summary>
32+
/// Converts this ordering specification to a SQL clause fragment.
33+
/// </summary>
34+
public string ToSql()
35+
{
36+
StringBuilder sb = new(ColumnName);
37+
38+
// Only append direction if explicitly set
39+
if (IsAscending.HasValue)
40+
{
41+
sb.Append(IsAscending.Value ? " ASC" : " DESC");
42+
}
43+
44+
// Only append NULLS clause if explicitly set
45+
if (NullsFirst.HasValue)
46+
{
47+
sb.Append(NullsFirst.Value ? " NULLS FIRST" : " NULLS LAST");
48+
}
49+
50+
return sb.ToString();
51+
}
52+
}
53+
54+
/// <summary>
55+
/// Fluent builder for creating OrderBy instances.
56+
/// </summary>
57+
public static class OrderByBuilder
58+
{
59+
/// <summary>
60+
/// Starts building an OrderBy specification for the specified property.
61+
/// </summary>
62+
/// <typeparam name="TEntity">The entity type containing the property.</typeparam>
63+
/// <param name="expression">A lambda expression selecting the property to order by.</param>
64+
public static OrderByConfiguration<TEntity> For<TEntity>(Expression<Func<TEntity, object>> expression) => new(expression);
65+
}
66+
67+
/// <summary>
68+
/// Fluent configuration for creating OrderBy instances.
69+
/// </summary>
70+
public class OrderByConfiguration<TEntity>(Expression<Func<TEntity, object>> expression)
71+
{
72+
private readonly string _propertyName = GetPropertyName(expression);
73+
74+
/// <summary>Creates an OrderBy using the database default direction.</summary>
75+
/// <param name="nullsFirst">Optional null sorting behavior. Null uses database default.</param>
76+
public OrderBy Default(bool? nullsFirst = null) => new(_propertyName, null, nullsFirst);
77+
78+
/// <summary>Creates an ascending OrderBy specification.</summary>
79+
/// <param name="nullsFirst">Optional null sorting behavior. Null uses database default.</param>
80+
public OrderBy Ascending(bool? nullsFirst = null) => new(_propertyName, true, nullsFirst);
81+
82+
/// <summary>Creates a descending OrderBy specification.</summary>
83+
/// <param name="nullsFirst">Optional null sorting behavior. Null uses database default.</param>
84+
public OrderBy Descending(bool? nullsFirst = null) => new(_propertyName, false, nullsFirst);
85+
86+
// Helper to extract the string name from the expression
87+
private static string GetPropertyName(Expression<Func<TEntity, object>> expression)
88+
{
89+
if (expression.Body is MemberExpression member) return member.Member.Name;
90+
if (expression.Body is UnaryExpression unary && unary.Operand is MemberExpression m) return m.Member.Name;
91+
throw new ArgumentException("Invalid expression. Please use a simple property access expression.");
92+
}
93+
}
94+
95+
/// <summary>
96+
/// Fluent builder for creating OrderBy instances using lambda expressions.
97+
/// </summary>
98+
/// <typeparam name="TEntity"></typeparam>
99+
public class OrderBySelector<TEntity>
100+
{
101+
/// <summary>Creates an OrderBy using the database default direction for the selected property.</summary>
102+
/// <param name="expression">A lambda expression selecting the property to order by.</param>
103+
/// <param name="nullsFirst">Optional null sorting behavior. Null uses database default.</param>
104+
public OrderBy By(Expression<Func<TEntity, object>> expression, bool? nullsFirst = null)
105+
=> new(GetPropertyName(expression), null, nullsFirst);
106+
107+
/// <summary>Creates an ascending OrderBy specification for the selected property.</summary>
108+
/// <param name="expression">A lambda expression selecting the property to order by.</param>
109+
/// <param name="nullsFirst">Optional null sorting behavior. Null uses database default.</param>
110+
public OrderBy ByAscending(Expression<Func<TEntity, object>> expression, bool? nullsFirst = null)
111+
=> new(GetPropertyName(expression), true, nullsFirst);
112+
113+
/// <summary>Creates a descending OrderBy specification for the selected property.</summary>
114+
/// <param name="expression">A lambda expression selecting the property to order by.</param>
115+
/// <param name="nullsFirst">Optional null sorting behavior. Null uses database default.</param>
116+
public OrderBy ByDescending(Expression<Func<TEntity, object>> expression, bool? nullsFirst = null)
117+
=> new(GetPropertyName(expression), false, nullsFirst);
118+
119+
private static string GetPropertyName(Expression<Func<TEntity, object>> expression)
120+
{
121+
if (expression.Body is MemberExpression m) return m.Member.Name;
122+
if (expression.Body is UnaryExpression u && u.Operand is MemberExpression m2) return m2.Member.Name;
123+
throw new ArgumentException("Expression must be a property access.");
124+
}
125+
}
126+
127+
/// <summary>
128+
/// Extension methods for creating OrderBy instances.
129+
/// </summary>
130+
public static class OrderByExtensions
131+
{
132+
/// <summary>
133+
/// Creates an ascending OrderBy instance.
134+
/// </summary>
135+
/// <param name="columnName">The name of the column to order by.</param>
136+
/// <param name="nullsFirst">Optional null sorting behavior. Null uses database default.</param>
137+
public static OrderBy Ascending(this string columnName, bool? nullsFirst = null)
138+
{
139+
return new OrderBy(columnName, true, nullsFirst);
140+
}
141+
142+
/// <summary>
143+
/// Creates a descending OrderBy instance.
144+
/// </summary>
145+
/// <param name="columnName">The name of the column to order by.</param>
146+
/// <param name="nullsFirst">Optional null sorting behavior. Null uses database default.</param>
147+
public static OrderBy Descending(this string columnName, bool? nullsFirst = null)
148+
{
149+
return new OrderBy(columnName, false, nullsFirst);
150+
}
151+
}
152+
}

0 commit comments

Comments
 (0)