|
| 1 | +--- |
| 2 | +name: 'sourceloop-audit-service' |
| 3 | +displayName: 'SourceLoop Audit Service' |
| 4 | +description: 'Track and record user actions with audit logging - inserts, updates, deletes with S3 archival and CSV export capabilities' |
| 5 | +keywords: |
| 6 | + [ |
| 7 | + 'audit', |
| 8 | + 'logging', |
| 9 | + 'audit-trail', |
| 10 | + 'compliance', |
| 11 | + 'tracking', |
| 12 | + 'sourceloop', |
| 13 | + 's3-archival', |
| 14 | + ] |
| 15 | +author: 'SourceFuse' |
| 16 | +--- |
| 17 | + |
| 18 | +# SourceLoop Audit Service |
| 19 | + |
| 20 | +## Overview |
| 21 | + |
| 22 | +A LoopBack 4 microservice for audit logging that tracks and records user actions including inserts, updates, and deletes. Supports S3 archival for long-term storage and CSV export capabilities. |
| 23 | + |
| 24 | +**Key capabilities:** |
| 25 | + |
| 26 | +- **Action Tracking**: Record inserts, updates, and deletes across services |
| 27 | +- **S3 Archival**: Archive audit logs to AWS S3 for compliance |
| 28 | +- **CSV Export**: Export audit data as CSV via ExcelJS |
| 29 | +- **Repository Mixin**: Drop-in audit logging for any repository |
| 30 | +- **Multi-source Retrieval**: Query from both database and archive |
| 31 | + |
| 32 | +## Available MCP Servers |
| 33 | + |
| 34 | +### sourceloop-cli |
| 35 | + |
| 36 | +**Package:** `@sourceloop/cli` |
| 37 | +**Connection:** Local stdio via npx |
| 38 | + |
| 39 | +Use the `microservice` tool with `--baseOnService --baseService=audit-service` to scaffold a new audit service instance. |
| 40 | + |
| 41 | +## Installation |
| 42 | + |
| 43 | +```typescript |
| 44 | +import {AuditServiceComponent} from '@sourceloop/audit-service'; |
| 45 | +import {BootMixin} from '@loopback/boot'; |
| 46 | +import {ApplicationConfig} from '@loopback/core'; |
| 47 | +import {RestApplication} from '@loopback/rest'; |
| 48 | + |
| 49 | +export class MyApplication extends BootMixin(RestApplication) { |
| 50 | + constructor(options: ApplicationConfig = {}) { |
| 51 | + super(options); |
| 52 | + |
| 53 | + // Validate required environment variables |
| 54 | + this.validateAuditEnv(); |
| 55 | + |
| 56 | + try { |
| 57 | + this.component(AuditServiceComponent); |
| 58 | + console.log('✅ Audit service loaded successfully'); |
| 59 | + } catch (error) { |
| 60 | + console.error('❌ Failed to initialize audit service:', error.message); |
| 61 | + throw error; |
| 62 | + } |
| 63 | + } |
| 64 | + |
| 65 | + private validateAuditEnv() { |
| 66 | + const required = ['DB_HOST', 'DB_PORT', 'DB_DATABASE']; |
| 67 | + const missing = required.filter(env => !process.env[env]); |
| 68 | + if (missing.length > 0) { |
| 69 | + throw new Error( |
| 70 | + `Missing required environment variables: ${missing.join(', ')}`, |
| 71 | + ); |
| 72 | + } |
| 73 | + |
| 74 | + // Validate S3 config if archival is enabled |
| 75 | + if (process.env.ENABLE_ARCHIVAL === 'true') { |
| 76 | + const s3Required = [ |
| 77 | + 'AWS_ACCESS_KEY_ID', |
| 78 | + 'AWS_SECRET_ACCESS_KEY', |
| 79 | + 'S3_BUCKET_NAME', |
| 80 | + ]; |
| 81 | + const s3Missing = s3Required.filter(env => !process.env[env]); |
| 82 | + if (s3Missing.length > 0) { |
| 83 | + throw new Error( |
| 84 | + `S3 archival enabled but missing: ${s3Missing.join(', ')}`, |
| 85 | + ); |
| 86 | + } |
| 87 | + } |
| 88 | + } |
| 89 | +} |
| 90 | +``` |
| 91 | + |
| 92 | +## Key Models |
| 93 | + |
| 94 | +- **AuditLog** - Core audit log entry with action, actor, timestamp, before/after data |
| 95 | +- **CustomFilter** - Custom query filters for audit retrieval |
| 96 | +- **MappingLog** - Entity mapping audit records |
| 97 | +- **Job** - Background archival job tracking |
| 98 | + |
| 99 | +## Key Controllers |
| 100 | + |
| 101 | +- **AuditController** - CRUD operations for audit logs, export, archival |
| 102 | + |
| 103 | +## Common Workflows |
| 104 | + |
| 105 | +### Workflow 1: Setup Audit Service |
| 106 | + |
| 107 | +```bash |
| 108 | +npx @sourceloop/cli microservice my-audit \ |
| 109 | + --baseOnService \ |
| 110 | + --baseService=audit-service \ |
| 111 | + --datasourceName=auditdb \ |
| 112 | + --datasourceType=postgresql \ |
| 113 | + --includeMigrations |
| 114 | + |
| 115 | +# After scaffolding, add these verification steps: |
| 116 | +cd my-audit |
| 117 | + |
| 118 | +# Test database connection |
| 119 | +npm run db:ping |
| 120 | + |
| 121 | +# Run migrations and verify |
| 122 | +npm run db:migrate |
| 123 | +npm run db:migrate:status |
| 124 | + |
| 125 | +# Verify service starts |
| 126 | +npm run build |
| 127 | +npm start & |
| 128 | +curl http://localhost:3000/ping |
| 129 | +``` |
| 130 | + |
| 131 | +**Critical validation checklist:** |
| 132 | + |
| 133 | +- ✅ Database connectivity verified |
| 134 | +- ✅ All migrations applied successfully |
| 135 | +- ✅ Service starts without errors |
| 136 | +- ✅ Health endpoint responds |
| 137 | + |
| 138 | +### Workflow 2: Add Audit Logging to a Repository |
| 139 | + |
| 140 | +Use the audit mixin from `@sourceloop/audit-log`: |
| 141 | + |
| 142 | +```typescript |
| 143 | +import {AuditRepositoryMixin} from '@sourceloop/audit-log'; |
| 144 | + |
| 145 | +export class MyEntityRepository extends AuditRepositoryMixin< |
| 146 | + MyEntity, |
| 147 | + typeof MyEntity.prototype.id, |
| 148 | + MyEntityRelations |
| 149 | +>(DefaultCrudRepository) { |
| 150 | + constructor(@inject('datasources.db') dataSource: DataSource) { |
| 151 | + super(MyEntity, dataSource); |
| 152 | + } |
| 153 | +} |
| 154 | +``` |
| 155 | + |
| 156 | +## Best Practices |
| 157 | + |
| 158 | +### Do: |
| 159 | + |
| 160 | +- Archive old audit logs to S3 for cost-effective long-term storage |
| 161 | +- Use the repository mixin for automatic audit logging |
| 162 | +- Index audit logs by actor, action type, and timestamp |
| 163 | +- Configure retention policies for compliance requirements |
| 164 | + |
| 165 | +### Don't: |
| 166 | + |
| 167 | +- Store sensitive data (passwords, tokens) in audit log details |
| 168 | +- Skip audit logging for delete operations |
| 169 | +- Query large audit datasets without filters - use pagination |
| 170 | + |
| 171 | +## Testing |
| 172 | + |
| 173 | +### Unit Tests |
| 174 | + |
| 175 | +```typescript |
| 176 | +import {createStubInstance, expect} from '@loopback/testlab'; |
| 177 | +import {AuditLogRepository} from '../repositories'; |
| 178 | +import {MyEntityRepository} from '../repositories'; |
| 179 | + |
| 180 | +describe('AuditRepositoryMixin', () => { |
| 181 | + let repository: MyEntityRepository; |
| 182 | + let auditRepo: sinon.SinonStubbedInstance<AuditLogRepository>; |
| 183 | + |
| 184 | + beforeEach(() => { |
| 185 | + auditRepo = createStubInstance(AuditLogRepository); |
| 186 | + repository = new MyEntityRepository(dataSource, auditRepo); |
| 187 | + }); |
| 188 | + |
| 189 | + it('should create audit log on entity creation', async () => { |
| 190 | + const entity = {name: 'Test Entity'}; |
| 191 | + await repository.create(entity); |
| 192 | + |
| 193 | + sinon.assert.calledOnce(auditRepo.create); |
| 194 | + sinon.assert.calledWith(auditRepo.create, sinon.match({ |
| 195 | + action: 'CREATE', |
| 196 | + entityName: 'MyEntity', |
| 197 | + })); |
| 198 | + }); |
| 199 | +}); |
| 200 | +``` |
| 201 | + |
| 202 | +### Integration Tests |
| 203 | + |
| 204 | +```typescript |
| 205 | +import {Client, expect} from '@loopback/testlab'; |
| 206 | +import {AuditApplication} from '../application'; |
| 207 | + |
| 208 | +describe('Audit Logs API', () => { |
| 209 | + let app: AuditApplication; |
| 210 | + let client: Client; |
| 211 | + |
| 212 | + before('setupApplication', async () => { |
| 213 | + ({app, client} = await setupApplication()); |
| 214 | + }); |
| 215 | + |
| 216 | + it('GET /audit-logs returns paginated results', async () => { |
| 217 | + const res = await client.get('/audit-logs').expect(200); |
| 218 | + |
| 219 | + expect(res.body).to.be.Array(); |
| 220 | + expect(res.headers).to.have.property('x-total-count'); |
| 221 | + }); |
| 222 | +}); |
| 223 | +``` |
| 224 | + |
| 225 | +### Testing Best Practices |
| 226 | + |
| 227 | +- Test audit log creation for all CRUD operations |
| 228 | +- Verify audit logs capture user identity (actedBy field) |
| 229 | +- Test audit log queries with pagination and filters |
| 230 | +- Ensure sensitive data is not logged in before/after fields |
| 231 | +- Test audit log retention and archival processes |
| 232 | + |
| 233 | +## Database |
| 234 | + |
| 235 | +Requires PostgreSQL. Run migrations: |
| 236 | + |
| 237 | +```bash |
| 238 | +npx db-migrate up --config database.json --migrations-dir migrations |
| 239 | +``` |
| 240 | + |
| 241 | +## Dependencies |
| 242 | + |
| 243 | +- `@sourceloop/core` |
| 244 | +- `@sourceloop/audit-log` |
| 245 | +- `exceljs` (CSV export) |
| 246 | +- `csvtojson` |
0 commit comments