This commit is contained in:
Eric Gullickson
2025-11-01 21:27:42 -05:00
parent 20953c6dee
commit 046c66fc7d
203 changed files with 5699 additions and 404943 deletions

View File

@@ -19,5 +19,5 @@
## Storage (`src/core/storage/`)
- `storage.service.ts` — Storage abstraction
- `adapters/minio.adapter.ts`MinIO S3-compatible adapter
- `adapters/filesystem.adapter.ts`Filesystem storage adapter

View File

@@ -16,7 +16,6 @@ const configSchema = z.object({
name: z.string(),
port: z.number(),
environment: z.string(),
tenant_id: z.string(),
node_env: z.string(),
}),
@@ -49,19 +48,9 @@ const configSchema = z.object({
url: z.string(),
timeout: z.string(),
}),
tenants: z.object({
url: z.string(),
timeout: z.string(),
}),
}),
}),
// MinIO configuration
minio: z.object({
endpoint: z.string(),
port: z.number(),
bucket: z.string(),
}),
// External APIs configuration
external: z.object({
@@ -85,7 +74,6 @@ const configSchema = z.object({
// Frontend configuration
frontend: z.object({
tenant_id: z.string(),
api_base_url: z.string(),
auth0: z.object({
domain: z.string(),
@@ -144,9 +132,6 @@ const configSchema = z.object({
// Secrets schema definition
const secretsSchema = z.object({
postgres_password: z.string(),
minio_access_key: z.string(),
minio_secret_key: z.string(),
platform_vehicles_api_key: z.string(),
auth0_client_secret: z.string(),
google_maps_api_key: z.string(),
});
@@ -162,8 +147,7 @@ export interface AppConfiguration {
getDatabaseUrl(): string;
getRedisUrl(): string;
getAuth0Config(): { domain: string; audience: string; clientSecret: string };
getPlatformServiceConfig(service: 'vehicles' | 'tenants'): { url: string; apiKey: string };
getMinioConfig(): { endpoint: string; port: number; accessKey: string; secretKey: string; bucket: string };
getPlatformVehiclesUrl(): string;
}
class ConfigurationLoader {
@@ -197,9 +181,6 @@ class ConfigurationLoader {
const secretFiles = [
'postgres-password',
'minio-access-key',
'minio-secret-key',
'platform-vehicles-api-key',
'auth0-client-secret',
'google-maps-api-key',
];
@@ -257,24 +238,8 @@ class ConfigurationLoader {
};
},
getPlatformServiceConfig(service: 'vehicles' | 'tenants') {
const serviceConfig = config.platform.services[service];
const apiKey = service === 'vehicles' ? secrets.platform_vehicles_api_key : 'mvp-platform-tenants-secret-key';
return {
url: serviceConfig.url,
apiKey,
};
},
getMinioConfig() {
return {
endpoint: config.minio.endpoint,
port: config.minio.port,
accessKey: secrets.minio_access_key,
secretKey: secrets.minio_secret_key,
bucket: config.minio.bucket,
};
getPlatformVehiclesUrl(): string {
return config.platform.services.vehicles.url;
},
};

View File

@@ -4,12 +4,10 @@
*/
import { Pool } from 'pg';
import { logger } from '../logging/logger';
import { getTenantConfig } from './tenant';
const tenant = getTenantConfig();
import { appConfig } from './config-loader';
export const pool = new Pool({
connectionString: tenant.databaseUrl,
connectionString: appConfig.getDatabaseUrl(),
max: 10,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 10000,

View File

@@ -4,11 +4,9 @@
*/
import Redis from 'ioredis';
import { logger } from '../logging/logger';
import { getTenantConfig } from './tenant';
import { appConfig } from './config-loader';
const tenant = getTenantConfig();
export const redis = new Redis(tenant.redisUrl, {
export const redis = new Redis(appConfig.getRedisUrl(), {
retryStrategy: (times) => Math.min(times * 50, 2000),
});

View File

@@ -1,69 +0,0 @@
import axios from 'axios';
import { appConfig } from './config-loader';
// Simple in-memory cache for tenant validation
const tenantValidityCache = new Map<string, { ok: boolean; ts: number }>();
const TENANT_CACHE_TTL_MS = 60_000; // 1 minute
/**
* Tenant-aware configuration for multi-tenant architecture
*/
export interface TenantConfig {
tenantId: string;
databaseUrl: string;
redisUrl: string;
platformServicesUrl: string;
isAdminTenant: boolean;
}
export const getTenantConfig = (): TenantConfig => {
const tenantId = appConfig.config.server.tenant_id;
const databaseUrl = tenantId === 'admin'
? appConfig.getDatabaseUrl()
: `postgresql://${appConfig.config.database.user}:${appConfig.secrets.postgres_password}@${tenantId}-postgres:5432/${appConfig.config.database.name}`;
const redisUrl = tenantId === 'admin'
? appConfig.getRedisUrl()
: `redis://${tenantId}-redis:6379`;
const platformServicesUrl = appConfig.getPlatformServiceConfig('tenants').url;
return {
tenantId,
databaseUrl,
redisUrl,
platformServicesUrl,
isAdminTenant: tenantId === 'admin'
};
};
export const isValidTenant = async (tenantId: string): Promise<boolean> => {
// Check cache
const now = Date.now();
const cached = tenantValidityCache.get(tenantId);
if (cached && (now - cached.ts) < TENANT_CACHE_TTL_MS) {
return cached.ok;
}
let ok = false;
try {
const baseUrl = appConfig.getPlatformServiceConfig('tenants').url;
const url = `${baseUrl}/api/v1/tenants/${encodeURIComponent(tenantId)}`;
const resp = await axios.get(url, { timeout: 2000 });
ok = resp.status === 200;
} catch { ok = false; }
tenantValidityCache.set(tenantId, { ok, ts: now });
return ok;
};
export const extractTenantId = (options: {
envTenantId?: string;
jwtTenantId?: string;
subdomain?: string;
}): string => {
const { envTenantId, jwtTenantId, subdomain } = options;
return envTenantId || jwtTenantId || subdomain || 'admin';
};

View File

@@ -1,84 +0,0 @@
/**
* Tenant detection and validation middleware for multi-tenant architecture
*/
import { FastifyRequest, FastifyReply } from 'fastify';
import { getTenantConfig, isValidTenant, extractTenantId } from '../config/tenant';
import { logger } from '../logging/logger';
// Extend FastifyRequest to include tenant context
declare module 'fastify' {
interface FastifyRequest {
tenantId: string;
tenantConfig: {
tenantId: string;
databaseUrl: string;
redisUrl: string;
platformServicesUrl: string;
isAdminTenant: boolean;
};
}
}
export const tenantMiddleware = async (
request: FastifyRequest,
reply: FastifyReply
) => {
try {
// Method 1: From environment variable (container-level)
const envTenantId = process.env.TENANT_ID;
// Method 2: From JWT token claims (verify or decode if available)
let jwtTenantId = (request as any).user?.['https://motovaultpro.com/tenant_id'] as string | undefined;
if (!jwtTenantId && typeof (request as any).jwtDecode === 'function') {
try {
const decoded = (request as any).jwtDecode();
jwtTenantId = decoded?.payload?.['https://motovaultpro.com/tenant_id']
|| decoded?.['https://motovaultpro.com/tenant_id'];
} catch { /* ignore decode errors */ }
}
// Method 3: From subdomain parsing (if needed)
const host = request.headers.host || '';
const subdomain = host.split('.')[0];
const subdomainTenantId = subdomain !== 'admin' && subdomain !== 'localhost' ? subdomain : undefined;
// Extract tenant ID with priority: Environment > JWT > Subdomain > Default
const tenantId = extractTenantId({
envTenantId,
jwtTenantId,
subdomain: subdomainTenantId
});
// Validate tenant exists
const isValid = await isValidTenant(tenantId);
if (!isValid) {
logger.warn('Invalid tenant access attempt', {
tenantId,
host,
path: request.url,
method: request.method
});
reply.code(403).send({ error: 'Invalid or unauthorized tenant' });
return;
}
// Get tenant configuration
const tenantConfig = getTenantConfig();
// Attach tenant context to request
request.tenantId = tenantId;
request.tenantConfig = tenantConfig;
logger.info('Tenant context established', {
tenantId,
isAdmin: tenantConfig.isAdminTenant,
path: request.url
});
return;
} catch (error) {
logger.error('Tenant middleware error', { error });
reply.code(500).send({ error: 'Internal server error' });
}
};

View File

@@ -0,0 +1,93 @@
import type { StorageService, HeadObjectResult, SignedUrlOptions, ObjectBody } from '../storage.service';
import * as fs from 'fs/promises';
import * as path from 'path';
import { createReadStream, createWriteStream } from 'fs';
import type { Readable } from 'stream';
import { pipeline } from 'stream/promises';
export class FilesystemAdapter implements StorageService {
private basePath: string;
constructor(basePath: string = '/app/data/documents') {
this.basePath = basePath;
}
async putObject(
_bucket: string,
key: string,
body: ObjectBody,
contentType?: string,
metadata?: Record<string, string>
): Promise<void> {
const filePath = path.join(this.basePath, key);
await fs.mkdir(path.dirname(filePath), { recursive: true });
if (Buffer.isBuffer(body) || typeof body === 'string') {
// For Buffer or string, write directly
await fs.writeFile(filePath, body);
} else {
// For Readable stream, pipe to file
const writeStream = createWriteStream(filePath);
await pipeline(body as Readable, writeStream);
}
// Store metadata in a sidecar file if provided
if (metadata || contentType) {
const metaPath = `${filePath}.meta.json`;
const meta: Record<string, string> = { ...(metadata || {}) };
if (contentType) {
meta['content-type'] = contentType;
}
await fs.writeFile(metaPath, JSON.stringify(meta, null, 2));
}
}
async getObjectStream(_bucket: string, key: string): Promise<Readable> {
const filePath = path.join(this.basePath, key);
return createReadStream(filePath);
}
async deleteObject(_bucket: string, key: string): Promise<void> {
const filePath = path.join(this.basePath, key);
await fs.unlink(filePath);
// Also delete metadata file if it exists
const metaPath = `${filePath}.meta.json`;
try {
await fs.unlink(metaPath);
} catch {
// Ignore if metadata file doesn't exist
}
}
async headObject(_bucket: string, key: string): Promise<HeadObjectResult> {
const filePath = path.join(this.basePath, key);
const stats = await fs.stat(filePath);
// Try to read metadata file
let contentType: string | undefined;
let metadata: Record<string, string> | undefined;
const metaPath = `${filePath}.meta.json`;
try {
const metaContent = await fs.readFile(metaPath, 'utf-8');
const meta = JSON.parse(metaContent);
contentType = meta['content-type'] || meta['Content-Type'];
metadata = meta;
} catch {
// Ignore if metadata file doesn't exist
}
return {
size: stats.size,
lastModified: stats.mtime,
contentType,
metadata,
};
}
async getSignedUrl(_bucket: string, key: string, _options?: SignedUrlOptions): Promise<string> {
// For filesystem storage, we don't use signed URLs
// The documents controller will handle serving the file directly
return `/api/documents/download/${key}`;
}
}

View File

@@ -1,66 +0,0 @@
import { Client as MinioClient } from 'minio';
import type { Readable } from 'stream';
import { appConfig } from '../../config/config-loader';
import type { HeadObjectResult, SignedUrlOptions, StorageService } from '../storage.service';
export function createMinioAdapter(): StorageService {
const { endpoint, port, accessKey, secretKey } = appConfig.getMinioConfig();
const client = new MinioClient({
endPoint: endpoint,
port,
useSSL: false,
accessKey,
secretKey,
});
const normalizeMeta = (contentType?: string, metadata?: Record<string, string>) => {
const meta: Record<string, string> = { ...(metadata || {}) };
if (contentType) meta['Content-Type'] = contentType;
return meta;
};
const adapter: StorageService = {
async putObject(bucket, key, body, contentType, metadata) {
const meta = normalizeMeta(contentType, metadata);
// For Buffer or string, size is known. For Readable, omit size for chunked encoding.
if (Buffer.isBuffer(body) || typeof body === 'string') {
await client.putObject(bucket, key, body as any, (body as any).length ?? undefined, meta);
} else {
await client.putObject(bucket, key, body as Readable, undefined, meta);
}
},
async getObjectStream(bucket, key) {
return client.getObject(bucket, key);
},
async deleteObject(bucket, key) {
await client.removeObject(bucket, key);
},
async headObject(bucket, key): Promise<HeadObjectResult> {
const stat = await client.statObject(bucket, key);
// minio types: size, etag, lastModified, metaData
return {
size: stat.size,
etag: stat.etag,
lastModified: stat.lastModified ? new Date(stat.lastModified) : undefined,
contentType: (stat.metaData && (stat.metaData['content-type'] || stat.metaData['Content-Type'])) || undefined,
metadata: stat.metaData || undefined,
};
},
async getSignedUrl(bucket, key, options?: SignedUrlOptions) {
const expires = Math.max(1, Math.min(7 * 24 * 3600, options?.expiresSeconds ?? 300));
if (options?.method === 'PUT') {
// MinIO SDK has presignedPutObject for PUT
return client.presignedPutObject(bucket, key, expires);
}
// Default GET
return client.presignedGetObject(bucket, key, expires);
},
};
return adapter;
}

View File

@@ -1,9 +1,9 @@
/**
* Provider-agnostic storage facade with S3-compatible surface.
* Initial implementation backed by MinIO using the official SDK.
* Provider-agnostic storage facade with filesystem storage.
* Uses local filesystem for document storage in /app/data/documents.
*/
import type { Readable } from 'stream';
import { createMinioAdapter } from './adapters/minio.adapter';
import { FilesystemAdapter } from './adapters/filesystem.adapter';
export type ObjectBody = Buffer | Readable | string;
@@ -38,11 +38,11 @@ export interface StorageService {
getSignedUrl(bucket: string, key: string, options?: SignedUrlOptions): Promise<string>;
}
// Simple factory — currently only MinIO; can add S3 in future without changing feature code
// Simple factory — uses filesystem storage
let singleton: StorageService | null = null;
export function getStorageService(): StorageService {
if (!singleton) {
singleton = createMinioAdapter();
singleton = new FilesystemAdapter('/app/data/documents');
}
return singleton;
}