YAML Loader

A comprehensive utility for loading and parsing YAML/JSON files with advanced $ref resolution, type safety, and performance optimizations.

Install

Install with NPM:

npm install @northern/di

or Yarn:

yarn add @northern/di

Table of Contents

- Overview
- Installation
- Basic Usage
- Type Safety
- Reference Resolution
- Configuration Options
- Error Handling
- Performance Features
- Security Features
- Advanced Usage
- API Reference
- Migration Guide
- Troubleshooting

Overview

The YAML loader module provides:

- Multi-format support: YAML and JSON file parsing
- Advanced reference resolution: Internal and external $ref resolution with circular detection
- Type safety: Full TypeScript generic support
- Performance: LRU caching and optimized algorithms
- Security: Configurable access controls and path validation
- Debugging: Comprehensive debug information and error reporting
- Extensibility: Custom resolver plugin system

Basic Usage

``typescript
import loadYaml from '@northern/yaml-loader';
`

$3

`typescript
// Basic usage - returns any type
const config = loadYaml('config.yaml');
console.log(config.name); // Access properties with basic autocomplete
`

$3

`typescript
// JSON files are supported automatically
const data = loadYaml('data.json');
`

$3

`typescript
// Complex nested YAML structures
const app = loadYaml('app.yaml');

// Access nested properties
const serverConfig = app.server;
const dbConfig = app.database;
`

Type Safety

$3

`typescript
interface AppConfig {
name: string;
version: number;
server: {
host: string;
port: number;
ssl?: boolean;
};
database: {
url: string;
pool: {
min: number;
max: number;
};
};
features: string[];
}

// Full type inference and compile-time checking
const config = loadYaml('app.yaml');

// TypeScript will catch type errors
const host: string = config.server.host; // ✅ Valid
const port: string = config.server.port; // ❌ TypeScript error: Type 'number' is not assignable to 'string'
config.invalidProperty; // ❌ TypeScript error: Property does not exist
`

$3

`typescript
interface User {
id: number;
name: string;
profile: {
email: string;
avatar?: string;
};
roles: Array<{
name: string;
permissions: string[];
}>;
}

interface UsersConfig {
users: User[];
metadata: {
created: Date;
version: string;
tags: string[];
};
}

const usersConfig = loadYaml('users.yaml');

// Full type safety throughout
usersConfig.users.forEach(user => {
console.log(user.profile.email); // ✅ Type-safe access
user.roles.forEach(role => {
console.log(role.permissions.join(', ')); // ✅ Type-safe array operations
});
});
`

$3

`typescript
type DatabaseConfig =
| { type: 'postgres'; host: string; port: number; database: string; }
| { type: 'mongodb'; url: string; collection: string; };

interface ServiceConfig {
name: string;
database: DatabaseConfig;
}

const config = loadYaml('service.yaml');

// TypeScript will help you handle different database types
if (config.database.type === 'postgres') {
console.log(config.database.host); // ✅ TypeScript knows this is postgres config
// config.database.url; // ❌ TypeScript error: Property 'url' does not exist
}
`

Reference Resolution

$3

`yaml


config.yaml


definitions:
server:
host: localhost
port: 3000
database:
url: postgresql://localhost:5432/mydb

services:
api:
# Internal reference to definitions
$ref: "#/definitions/server"
database:
$ref: "#/definitions/database"
`

`typescript
const config = loadYaml('config.yaml');
// Result will have fully resolved structure
// {
// definitions: { server: {...}, database: {...} },
// services: {
// api: { host: 'localhost', port: 3000 },
// database: { url: 'postgresql://localhost:5432/mydb' }
// }
// }
`

$3

`yaml


shared/database.yaml

url: postgresql://localhost:5432/mydb
pool:
min: 5
max: 20

config.yaml

database:
$ref: "./shared/database.yaml"

api:
version: v1
port: 8080
`

$3

`yaml


schemas/user.yaml

definitions:
User:
type: object
properties:
id:
type: integer
name:
type: string
email:
type: string
format: email

api.yaml

userSchema:
$ref: "./schemas/user.yaml#/definitions/User"

endpoints:
- path: /users
schema:
$ref: "./schemas/user.yaml#/definitions/User"
`

$3

`yaml


level3.yaml

finalValue: "Deeply nested reference resolved"

level2.yaml

data:
$ref: "./level3.yaml"

level1.yaml

nested:
$ref: "./level2.yaml"

main.yaml

result:
$ref: "./level1.yaml"
`

$3

`yaml


item.yaml

type: shared-item
properties:
name: string
value: number

main.yaml

items:
- name: inline1
type: local
- $ref: "./item.yaml"
- name: inline2
type: local
- $ref: "./item.yaml"
`

$3

`yaml


data.yaml

"a/b path":
value: "path with slash"
"a~b":
value: "path with tilde"
items:
- first
- second
- third

ref.yaml

Reference with escaped characters

slashExample:
$ref: "./data.yaml#/a~1b path"
tildeExample:
$ref: "./data.yaml#/a~0b"
arrayItem:
$ref: "./data.yaml#/items/1"
`

Configuration Options

$3

`typescript
import { YamlLoaderOptions } from './yaml-loader';

const options: YamlLoaderOptions = {
maxCacheSize: 50,
allowExternalAccess: false,
strictMode: true
};

const config = loadYaml('config.yaml', options);
`

$3

`typescript
// Large cache for enterprise applications
const enterpriseOptions: YamlLoaderOptions = {
maxCacheSize: 500 // Cache up to 500 files
};

// Small cache for simple applications
const simpleOptions: YamlLoaderOptions = {
maxCacheSize: 10 // Cache only 10 files
};
`

$3

`typescript
// Default: prevent directory traversal
const secureOptions: YamlLoaderOptions = {
allowExternalAccess: false
};

// Allow external access for trusted environments
const openOptions: YamlLoaderOptions = {
allowExternalAccess: true
};
`

$3

`typescript
// Enable strict validation
const strictOptions: YamlLoaderOptions = {
strictMode: true
};

// Disable for legacy compatibility
const lenientOptions: YamlLoaderOptions = {
strictMode: false
};
`

Error Handling

$3

`typescript
import { YamlLoaderError } from './yaml-loader';

try {
const config = loadYaml('config.yaml');
console.log('Loaded successfully:', config);
} catch (error) {
if (error instanceof YamlLoaderError) {
console.error(YAML Error (${error.type}): ${error.message});
if (error.path) {
console.error('Path:', error.path);
}
if (error.refChain) {
console.error('Reference chain:', error.refChain.join(' -> '));
}
} else {
console.error('Unexpected error:', error);
}
}
`

$3

`typescript
// Circular reference error
try {
loadYaml('circular.yaml');
} catch (error) {
if (error instanceof YamlLoaderError && error.type === 'circular_ref') {
console.log('Circular reference detected');
console.log('Chain:', error.refChain);
}
}

// File not found error
try {
loadYaml('missing.yaml');
} catch (error) {
if (error instanceof YamlLoaderError && error.type === 'file_not_found') {
console.log('File not found:', error.path);
}
}

// Invalid JSON pointer
try {
loadYaml('invalid-pointer.yaml');
} catch (error) {
if (error instanceof YamlLoaderError && error.type === 'invalid_pointer') {
console.log('Invalid pointer path:', error.path);
}
}

// Parse error
try {
loadYaml('invalid.yaml');
} catch (error) {
if (error instanceof YamlLoaderError && error.type === 'parse_error') {
console.log('Parse error:', error.message);
}
}
`

$3

`typescript
import { validateYamlReferences } from './yaml-loader';

// Validate without full resolution
const validation = validateYamlReferences('config.yaml');

if (validation.isValid) {
console.log('All references are valid');
const config = loadYaml('config.yaml');
} else {
console.log('Validation failed:');
validation.errors.forEach(error => {
console.error(- ${error.type}: ${error.message});
});
validation.warnings.forEach(warning => {
console.warn(Warning: ${warning});
});
}
`

Performance Features

$3

`typescript
// Configure cache size based on application needs
const options: YamlLoaderOptions = {
maxCacheSize: 100 // Default: 100 files
};

// Cache automatically handles:
// - Most recently used files stay in memory
// - Least recently used files are evicted when limit reached
// - Multiple references to same file use cached version

const config = loadYaml('main.yaml', options);
// If main.yaml references shared.yaml multiple times, it's loaded once
`

$3

`typescript
import { loadYamlWithDebug } from './yaml-loader';

const { result, debug } = loadYamlWithDebug('complex.yaml');

console.log(Resolution completed in ${debug.resolutionTime}ms);
console.log(Processed ${debug.refChain.length} references);
console.log(Cached ${debug.fileCache.size} files);

// Analyze cache contents
for (const [file, type] of debug.fileCache) {
console.log(${file}: ${type});
}
`

$3

`typescript
// Create a performance wrapper
function loadWithMetrics(filename: string): T {
const start = Date.now();
const { result, debug } = loadYamlWithDebug(filename);
const duration = Date.now() - start;

console.log(Load time: ${duration}ms);
console.log(Cache hits: ${debug.fileCache.size});
console.log(References resolved: ${debug.refChain.length});

return result;
}

const config = loadWithMetrics('config.yaml');
`

Security Features

$3

`typescript
// Default: prevents access outside base directory
const secureConfig = loadYaml('config.yaml');
// This will fail if config.yaml contains: $ref: "../secret.yaml"
`

$3

`typescript
// Enable external access for trusted environments
const options: YamlLoaderOptions = {
allowExternalAccess: true
};

const config = loadYaml('config.yaml', options);
// Now allows: $ref: "../shared/config.yaml"
`

$3

`typescript
// Custom path validation
function loadWithValidation(filename: string, allowedPaths: string[]) {
try {
return loadYaml(filename, { allowExternalAccess: true });
} catch (error) {
if (error instanceof YamlLoaderError && error.type === 'file_not_found') {
// Check if path is in allowed list
const isAllowed = allowedPaths.some(path =>
error.path?.includes(path)
);
if (!isAllowed) {
throw new Error('Access denied to external file');
}
}
throw error;
}
}
`

Advanced Usage

$3

`typescript
import { YamlLoaderBuilder } from './yaml-loader';

// Simple builder
const loader = new YamlLoaderBuilder()
.withCache(50)
.withStrictMode(true)
.withExternalAccess(false)
.build();

const config = loader('config.yaml');

// Generic builder with type safety
interface AppConfig {
name: string;
version: number;
}

const typedLoader = new YamlLoaderBuilder()
.withCache(25)
.buildGeneric();

const appConfig = typedLoader('app.yaml');
// Full type inference with AppConfig interface
`

$3

`typescript
import { YamlLoaderBuilder } from './yaml-loader';

// Environment variable resolver
const loader = new YamlLoaderBuilder()
.withCustomResolver('env:', (ref) => {
const varName = ref.replace('env:', '');
return process.env[varName] || '';
})
.build();

// Usage in YAML:


config.yaml

database:
url: env:DATABASE_URL
password: env:DB_PASSWORD

const config = loader('config.yaml');
// config.database.url will contain the environment variable value
`

$3

`typescript
import axios from 'axios';

// HTTP-based resolver for remote schemas
const httpLoader = new YamlLoaderBuilder()
.withCustomResolver('http:', async (ref) => {
const response = await axios.get(ref);
return response.data;
})
.withCustomResolver('https:', async (ref) => {
const response = await axios.get(ref);
return response.data;
})
.build();

// Usage in YAML:


config.yaml


schema:
$ref: "https://example.com/schemas/user.json"
`

$3

`typescript
// Template-based resolver for dynamic values
const templateLoader = new YamlLoaderBuilder()
.withCustomResolver('template:', (ref) => {
const templateName = ref.replace('template:', '');
const templates = {
'user-service': {
port: 3000,
endpoints: ['/users', '/users/{id}']
},
'auth-service': {
port: 3001,
endpoints: ['/login', '/logout', '/refresh']
}
};
return templates[templateName];
})
.build();

// Usage in YAML:


config.yaml


userService:
$ref: "template:user-service"
authService:
$ref: "template:auth-service"
`

$3

`typescript
function loadConfig(configPath: string, isProduction: boolean) {
const options: YamlLoaderOptions = {
maxCacheSize: isProduction ? 500 : 50,
allowExternalAccess: !isProduction,
strictMode: isProduction
};

return loadYaml(configPath, options);
}

const devConfig = loadConfig('config.yaml', false);
const prodConfig = loadConfig('config.yaml', true);
`

$3

`typescript
// Create a plugin loader
interface Plugin {
name: string;
resolver: (ref: string) => any;
}

class ExtensibleYamlLoader {
private builder = new YamlLoaderBuilder();

addPlugin(plugin: Plugin): this {
this.builder.withCustomResolver(${plugin.name}:, plugin.resolver);
return this;
}

build() {
return this.builder.buildGeneric();
}
}

// Usage
const loader = new ExtensibleYamlLoader()
.addPlugin({
name: 'env',
resolver: (ref) => process.env[ref.replace('env:', '')] || ''
})
.addPlugin({
name: 'secret',
resolver: (ref) => {
// Load from secret manager
return getSecret(ref.replace('secret:', ''));
}
})
.build();
`

API Reference

$3

#### loadYaml(filename: string, options?: YamlLoaderOptions): T

Loads and parses a YAML file with reference resolution.

Parameters:
- filename: string - Absolute path to the YAML/JSON file
- options?: YamlLoaderOptions - Configuration options (optional)

Returns:
- T - Parsed and resolved content with type inference

Example:
`typescript
const config = loadYaml('config.yaml');
`

#### loadYamlWithDebug(filename: string, options?: YamlLoaderOptions): { result: T; debug: DebugInfo }

Loads YAML with debug information for troubleshooting.

Returns:
- result: T - The loaded configuration
- debug: DebugInfo - Debug information including cache stats and timing

#### validateYamlReferences(filename: string, options?: YamlLoaderOptions): ValidationResult

Validates references without full resolution.

Returns:
- isValid: boolean - Whether all references are valid
- errors: YamlLoaderError[] - List of validation errors
- warnings: string[] - List of warnings

$3

#### YamlLoaderBuilder

Builder pattern for creating configured loaders.

Methods:
- withCache(size: number): this - Set cache size
- withStrictMode(enabled: boolean): this - Enable/disable strict mode
- withExternalAccess(enabled: boolean): this - Allow/deny external access
- withCustomResolver(prefix: string, resolver: (ref: string) => any): this - Add custom resolver
- build(): (filename: string) => any - Build configured loader
- buildGeneric(): (filename: string) => T - Build typed loader

#### YamlLoaderError

Enhanced error class for YAML loading issues.

Properties:
- type: 'circular_ref' | 'file_not_found' | 'invalid_pointer' | 'parse_error' - Error category
- path?: string - File path or reference path
- refChain?: string[] - Chain of references that led to the error

$3

#### YamlLoaderOptions

Configuration options for YAML loading.

`typescript
interface YamlLoaderOptions {
maxCacheSize?: number; // Maximum files to cache (default: 100)
allowExternalAccess?: boolean; // Allow directory traversal (default: false)
customResolvers?: Map any>; // Custom resolvers
strictMode?: boolean; // Enable strict validation (default: false)
}
`

#### DebugInfo

Debug information from loadYamlWithDebug.

`typescript
interface DebugInfo {
refChain: string[]; // Reference resolution chain
fileCache: Map; // Cached files and their types
resolutionTime: number; // Time in milliseconds
}
`

Migration Guide

$3

Before:
`typescript
const yamlLoader = require('./yaml-loader');
const config = yamlLoader('config.yaml');
`

After:
`typescript
import { loadYaml } from './yaml-loader';
const config = loadYaml('config.yaml');
`

$3

Before:
`typescript
const config = loadYaml('config.yaml');
// No type checking
const port = config.port; // Could be anything
`

After:
`typescript
interface Config {
port: number;
host: string;
}
const config = loadYaml('config.yaml');
const port = config.port; // TypeScript knows it's a number
`

$3

Before:
`typescript
const config = loadYaml('config.yaml');
`

After:
`typescript
const config = loadYaml('config.yaml', {
maxCacheSize: 50,
allowExternalAccess: true,
strictMode: false
});
`

$3

Before:
`typescript
const config = loadYaml('config.yaml');
`

After:
`typescript
const loader = new YamlLoaderBuilder()
.withCache(50)
.withStrictMode(true)
.build();

const config = loader('config.yaml');
`

Troubleshooting

$3

#### Circular References

Error: Circular reference detected: A -> B -> A

Solution:
- Check reference chains in YAML files
- Use validateYamlReferences() to detect issues early
- Consider restructuring to avoid circular dependencies

#### File Not Found

Error: Failed to load file: /path/to/missing.yaml

Solutions:
- Verify file paths are correct
- Check if allowExternalAccess: true is needed for external files
- Use absolute paths for reliable resolution

#### Type Errors

Error: TypeScript compilation errors

Solutions:
- Define proper interfaces for your YAML structure
- Use generic type parameter: loadYaml()
- Check for optional properties with ? in interfaces

#### Performance Issues

Symptoms: Slow loading times

Solutions:
- Increase cache size with maxCacheSize
- Use debug mode to identify bottlenecks
- Consider simplifying reference chains

$3

#### Using Debug Mode

`typescript
const { result, debug } = loadYamlWithDebug('complex.yaml');

console.log('Performance Analysis:');
console.log(- Total time: ${debug.resolutionTime}ms);
console.log(- References: ${debug.refChain.length});
console.log(- Cache size: ${debug.fileCache.size});

console.log('Reference Chain:');
debug.refChain.forEach((ref, index) => {
console.log(${index + 1}. ${ref});
});
`

#### Validation Before Loading

`typescript
const validation = validateYamlReferences('config.yaml');
if (!validation.isValid) {
console.log('Issues found:');
validation.errors.forEach(error => {
console.error(- ${error.type}: ${error.message});
});
return;
}

// Only load if validation passes
const config = loadYaml('config.yaml');
`

#### File Analysis

`typescript
// Create a file usage analyzer
function analyzeFileUsage(filename: string) {
const { debug } = loadYamlWithDebug(filename);

const fileUsage = new Map();

// Count file usage from reference chain
debug.refChain.forEach(ref => {
const filePath = ref.split('#')[0];
fileUsage.set(filePath, (fileUsage.get(filePath) || 0) + 1);
});

console.log('File Usage:');
for (const [file, count] of fileUsage) {
console.log(${file}: ${count} references);
}
}

analyzeFileUsage('config.yaml');
`

$3

#### Cache Tuning

`typescript
// For small applications
const smallAppOptions = { maxCacheSize: 10 };

// For medium applications
const mediumAppOptions = { maxCacheSize: 50 };

// For large applications
const largeAppOptions = { maxCacheSize: 200 };
`

#### Reference Optimization

`yaml


Instead of many small references

api:
user:
$ref: "./schemas/user.yaml"
product:
$ref: "./schemas/product.yaml"
order:
$ref: "./schemas/order.yaml"

Consider consolidating

api:
schemas:
$ref: "./schemas/all.yaml"
`

#### Lazy Loading

`typescript
// Load only what you need
function loadSection(filename: string, pointer: string): T {
const fullPath = ${filename}#${pointer};
return loadYaml(fullPath);
}

const userConfig = loadSection('config.yaml', '#/services/user');
const dbConfig = loadSection('config.yaml', '#/database');
``