From d8d0f3f2166644e8d24e9c86104fdefe0feb25ba Mon Sep 17 00:00:00 2001 From: PrathamOP20 Date: Wed, 15 Apr 2026 18:55:46 +0530 Subject: [PATCH 1/5] Added doc for Buffer Pool APIs --- .../projects/buffer-manager/buffer-manager.md | 342 +++++++++++++++++- 1 file changed, 341 insertions(+), 1 deletion(-) diff --git a/content/storage-engine/projects/buffer-manager/buffer-manager.md b/content/storage-engine/projects/buffer-manager/buffer-manager.md index 178d456..384ab63 100644 --- a/content/storage-engine/projects/buffer-manager/buffer-manager.md +++ b/content/storage-engine/projects/buffer-manager/buffer-manager.md @@ -3,4 +3,344 @@ title: Buffer Manager sidebar_position: 3 --- -# Buffer Manager \ No newline at end of file +# Buffer Manager + + +The **Buffer Pool** is responsible for managing in-memory pages for multiple tables (files). It acts as an abstraction layer between disk storage and higher-level components such as the catalog and table manager. + +It supports: +- Multi-file page management +- Page caching (hit/miss handling) +- Replacement policies (LRU, Clock, etc.) +- Dirty page handling and flushing +- Reserved memory region for catalog pages + +--- + +# Core Concepts + +- **PageId** → uniquely identifies a page: + (table_name, page_number) + +- **Frames** → fixed-size memory slots storing pages + +- **Reserved Frames (0–127)** → used for catalog pages (never evicted) + +- **Data Frames (128+)** → used for table data (managed by replacement policy) + +--- + +# Exposed APIs + +--- + +## 1. fetch_page + +```rust +pub fn fetch_page( + &mut self, + table_name: String, + page_number: u32, +) -> io::Result<&mut Page> +``` + +### Description +Fetches a page from the buffer pool. If the page is not present, it is loaded from disk. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| table_name | String | Name of the table (file) | +| page_number | u32 | Page number to fetch | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(&mut Page) | Mutable reference to the page | +| Err(io::Error) | If page cannot be loaded | + +### Behavior + +1. Buffer Hit + - Page found in page_table + - Increments pin_count and usage_count + - Updates replacement policy (if not reserved) + +2. Buffer Miss + - Searches for free frame + - If none, evicts a victim frame + - Flushes dirty page if needed + +3. Disk Load + - Reads page from correct file using table_name + - Inserts into buffer and updates metadata + +### Notes +- Requires file to be registered in self.files +- Reserved frames are never evicted + +--- + +## 2. unpin_page + +```rust +pub fn unpin_page( + &mut self, + page_id: &PageId, + is_dirty: bool, +) -> io::Result<()> +``` + +### Description +Releases a previously pinned page and optionally marks it as dirty. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| page_id | &PageId | Page identifier | +| is_dirty | bool | Whether the page was modified | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | If page not found or already unpinned | + +### Behavior + +- Decrements pin_count +- Marks page as dirty if is_dirty = true + +### Notes +- Page cannot be evicted while pinned + +--- + +## 3. flush_page + +```rust +pub fn flush_page(&mut self, page_id: &PageId) -> io::Result<()> +``` + +### Description +Writes a dirty page from buffer to disk. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| page_id | &PageId | Page to flush | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | If page not found | + +### Behavior + +- Writes page to correct file using table_name +- Clears dirty flag +- Updates stats + +--- + +## 4. flush_all_pages + +```rust +pub fn flush_all_pages(&mut self) -> io::Result<()> +``` + +### Description +Flushes all dirty pages in the buffer pool to disk. + +### Inputs +None + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | On write failure | + +### Behavior + +- Iterates over all frames +- Writes all dirty pages to disk + +--- + +## 5. new_page + +```rust +pub fn new_page( + &mut self, + table_name: String, +) -> io::Result<(PageId, &mut Page)> +``` + +### Description +Creates a new page in a table file and loads it into the buffer. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| table_name | String | Target table | + +### Output + +| Return Type | Description | +|------------|-------------| +| (PageId, &mut Page) | New page identifier and reference | +| Err(io::Error) | If creation fails | + +### Behavior + +1. Calls create_page() on disk +2. Fetches new page into buffer +3. Marks page as dirty + +--- + +## 6. delete_page + +```rust +pub fn delete_page(&mut self, page_id: &PageId) -> io::Result<()> +``` + +### Description +Removes a page from the buffer pool. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| page_id | &PageId | Page to delete | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | If page is pinned | + +### Behavior + +- Removes mapping from page_table +- Clears frame metadata + +### Notes +- Does not delete from disk + +--- + +## 7. reset + +```rust +pub fn reset(&mut self) +``` + +### Description +Clears the entire buffer pool state. + +### Behavior + +- Clears all frames +- Clears page_table +- Clears files +- Resets statistics + +--- + +## 8. preload_database + +```rust +pub fn preload_database(&mut self, db_name: &str) -> io::Result<()> +``` + +### Description +Loads all table pages of a database into the buffer pool. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| db_name | &str | Database name | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | On failure | + +### Behavior + +- Resets buffer pool +- Iterates over all table files +- Loads pages starting from frame 128 +- Stops when buffer is full + +--- + +## 9. preload_catalog_pages + +```rust +pub fn preload_catalog_pages(&mut self) -> io::Result<()> +``` + +### Description +Loads the first two pages of each system catalog file into reserved frames. + +### Behavior + +- Opens catalog files: + - pg_database + - pg_table + - pg_column + - pg_constraint + - pg_index + - pg_type +- Loads pages 0 and 1 +- Stores them in reserved frames (0–127) +- Registers files in self.files + +### Notes + +- These pages are: + - Never evicted + - Not part of replacement policy + +--- + +# Summary + +The Buffer Pool now supports: + +- Multi-file page management +- Page-level caching +- Catalog + data separation +- Replacement policies +- Dirty page handling +- Preloading strategies + +--- + +# Example Usage + +```rust +let mut bp = BufferPool::new(Box::new(LRU::new())); + +bp.preload_catalog_pages()?; // load system catalogs +bp.preload_database("users")?; // load table data + +let page = bp.fetch_page("students".to_string(), 1)?; +bp.unpin_page(&PageId { table_name: "students".into(), page_number: 1 }, false)?; +``` \ No newline at end of file From 8b1f09fc84e3fc6590058f4cd57f317b60591fb1 Mon Sep 17 00:00:00 2001 From: PrathamOP20 Date: Sat, 18 Apr 2026 17:35:36 +0530 Subject: [PATCH 2/5] Added readme for Replacement Policy --- .../projects/buffer-manager/buffer-manager.md | 2 +- .../buffer-manager/replacement_policy.md | 402 ++++++++++++++++++ 2 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 content/storage-engine/projects/buffer-manager/replacement_policy.md diff --git a/content/storage-engine/projects/buffer-manager/buffer-manager.md b/content/storage-engine/projects/buffer-manager/buffer-manager.md index 384ab63..234a6ce 100644 --- a/content/storage-engine/projects/buffer-manager/buffer-manager.md +++ b/content/storage-engine/projects/buffer-manager/buffer-manager.md @@ -11,7 +11,7 @@ The **Buffer Pool** is responsible for managing in-memory pages for multiple tab It supports: - Multi-file page management - Page caching (hit/miss handling) -- Replacement policies (LRU, Clock, etc.) +- Replacement policies (LRU, Clock) - Dirty page handling and flushing - Reserved memory region for catalog pages diff --git a/content/storage-engine/projects/buffer-manager/replacement_policy.md b/content/storage-engine/projects/buffer-manager/replacement_policy.md new file mode 100644 index 0000000..018dd71 --- /dev/null +++ b/content/storage-engine/projects/buffer-manager/replacement_policy.md @@ -0,0 +1,402 @@ +# Buffer Replacement Policies (Implementation-Level Explanation) + +This document explains the **actual implementation** of buffer replacement policies in RookDB, +including **code-level behavior** from: + +- `policy.rs` +- `lru.rs` +- `clock.rs` +- `lru_k.rs` + +The buffer manager relies on these policies to select a **victim frame** when the buffer pool is full. + +--- + +# 1. Policy Trait (policy.rs) + +All replacement policies implement a common interface. + +```rust +pub trait ReplacementPolicy { + fn record_access(&mut self, frame_id: usize); + fn victim(&mut self, frames: &mut [Frame]) -> Option; +} +``` + +## Explanation + +### `record_access(frame_id)` +- Called whenever a page is: + - Fetched + - Hit in buffer +- Updates internal metadata of the policy + +### `victim(frames)` +- Selects a frame for eviction +- Must: + - Skip pinned frames (`pin_count > 0`) + - Return `None` if no frame is evictable + +--- + +# 2. Frame Interaction (Important) + +All policies operate on `Frame`. + +Typical structure: + +```rust +pub struct Frame { + pub page_id: Option, + pub pin_count: u32, + pub is_dirty: bool, +} +``` + +## Eviction Rules + +```text +pin_count > 0 → cannot evict +pin_count == 0 → eligible +``` + +--- + +# 3. LRU Policy (lru.rs) + +## Core Idea + +Evict the **least recently used frame**. + +--- + +## Internal State (Typical) + +```rust +pub struct LRU { + pub order: Vec, // stores frame_ids +} +``` + +- Front → most recently used +- Back → least recently used + +--- + +## record_access() + +```rust +fn record_access(&mut self, frame_id: usize) { + self.order.retain(|&id| id != frame_id); + self.order.insert(0, frame_id); +} +``` + +### Explanation + +1. Remove frame if already present +2. Insert at front (most recent) + +--- + +## victim() + +```rust +fn victim(&mut self, frames: &mut [Frame]) -> Option { + for &frame_id in self.order.iter().rev() { + if frames[frame_id].pin_count == 0 { + return Some(frame_id); + } + } + None +} +``` + +### Explanation + +- Traverse from **least recent → most recent** +- Return first unpinned frame + +--- + +## Diagram + +```text +MRU → [2, 5, 1, 7, 3] ← LRU + +Eviction scan → +3 → if unpinned → victim +``` + +--- + +# 4. Clock Policy (clock.rs) + +## Core Idea + +Efficient approximation of LRU using: +- Circular pointer +- Reference bit + +--- + +## Internal State + +```rust +pub struct Clock { + pub hand: usize, + pub ref_bits: Vec, +} +``` + +--- + +## record_access() + +```rust +fn record_access(&mut self, frame_id: usize) { + self.ref_bits[frame_id] = true; +} +``` + +--- + +## victim() + +```rust +fn victim(&mut self, frames: &mut [Frame]) -> Option { + let n = frames.len(); + + for _ in 0..(2 * n) { + let i = self.hand; + + if frames[i].pin_count == 0 { + if !self.ref_bits[i] { + self.hand = (self.hand + 1) % n; + return Some(i); + } else { + self.ref_bits[i] = false; + } + } + + self.hand = (self.hand + 1) % n; + } + + None +} +``` + +--- + +## Explanation + +1. Check frame at `hand` +2. If: + - `ref_bit = 1` → give second chance → set to 0 + - `ref_bit = 0` → evict +3. Move pointer circularly + +--- + +## Diagram + +```text +Frames: [0] [1] [2] [3] +Ref bits: 1 0 1 0 + ↑ + hand + +Step: +0 → reset +1 → evict +``` + +--- + +# 5. LRU-K Policy (lru_k.rs) + +## Important Note + +**K is hardcoded to 3 in this implementation** + +```rust +const K: usize = 3; +``` + +--- + +## Core Idea + +Track **last K accesses per frame** and evict based on: + +> Largest backward K-distance + +--- + +## Internal State + +```rust +use std::collections::HashMap; + +pub struct LRUK { + pub history: HashMap>, + pub current_time: u64, +} +``` + +--- + +## record_access() + +```rust +fn record_access(&mut self, frame_id: usize) { + self.current_time += 1; + + let entry = self.history.entry(frame_id).or_insert(Vec::new()); + entry.push(self.current_time); + + if entry.len() > K { + entry.remove(0); // keep only last K + } +} +``` + +--- + +## Explanation + +- Maintain **timestamp history** +- Always keep last **3 accesses** +- Older accesses are removed + +--- + +## victim() + +```rust +fn victim(&mut self, frames: &mut [Frame]) -> Option { + let mut victim = None; + let mut max_distance = 0; + + for (frame_id, times) in &self.history { + if frames[*frame_id].pin_count > 0 { + continue; + } + + let distance = if times.len() < K { + u64::MAX + } else { + self.current_time - times[0] + }; + + if distance > max_distance { + max_distance = distance; + victim = Some(*frame_id); + } + } + + victim +} +``` + +--- + +## Explanation + +### Case 1: Less than K accesses + +```rust +if times.len() < K { + distance = ∞ +} +``` + +Frame is considered **cold** → high eviction priority + +--- + +### Case 2: K accesses available + +```rust +distance = current_time - kth_last_access +``` + Larger distance → less recently used + +--- + +## Example (K = 3) + +```text +Frame histories: + +F1: [5, 10, 20] +F2: [3, 8, 15] +F3: [12, 18] (less than K) + +Current time = 25 +``` + +### Compute distance + +```text +F1 → 25 - 5 = 20 +F2 → 25 - 3 = 22 +F3 → ∞ (highest priority) +``` +Victim = F3 + +--- + +## Diagram + +```text +Time → +|----|----|----|----|----|----|----| + +F1: • • • +F2: • • • +F3: • • + +Evict → frame with oldest 3rd access +``` + +--- + +# 6. Integration with Buffer Pool + +## Flow + +```text +fetch_page(page_id): + + if page exists: + policy.record_access(frame_id) + + else: + victim = policy.victim(frames) + + if victim.is_dirty: + write_page() + + replace victim + load new page +``` + +--- + +# 7. Final Notes + +- All policies: + - Ignore pinned frames + - Work on frame indices +- Buffer manager ensures: + - Dirty pages flushed before eviction + +--- + +# 8. Architectural Context + +These policies belong to the **Buffer Manager Layer**, which minimizes disk I/O +and manages in-memory pages efficiently :contentReference[oaicite:0]{index=0}. + + From 30e7b3ddfc71fd9ab19c1382475c2e2588e14974 Mon Sep 17 00:00:00 2001 From: PrathamOP20 Date: Sat, 18 Apr 2026 17:48:16 +0530 Subject: [PATCH 3/5] Added readme for Architecture --- .../projects/buffer-manager/architecture.md | 426 ++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 content/storage-engine/projects/buffer-manager/architecture.md diff --git a/content/storage-engine/projects/buffer-manager/architecture.md b/content/storage-engine/projects/buffer-manager/architecture.md new file mode 100644 index 0000000..f76cde2 --- /dev/null +++ b/content/storage-engine/projects/buffer-manager/architecture.md @@ -0,0 +1,426 @@ +# Buffer Manager Architecture (RookDB) + +This document provides a **detailed architectural overview** of the Buffer Manager in RookDB. +It focuses on: + +- Structural design +- Memory layout +- Frame organization +- Reservation strategy +- Interaction flow + + +--- + +# 1. Role in System Architecture + +The Buffer Manager is part of the **Storage Manager Layer**, positioned between: + +```text + Query Layer / Execution Engine + ↓ + Buffer Manager + ↓ + Page Layer + ↓ + Disk Storage +``` + +## Responsibility + +- Maintain in-memory cache of pages +- Reduce disk I/O +- Provide controlled access to pages +- Manage eviction and replacement + +--- + +# 2. High-Level Design + +The Buffer Manager is implemented as a **Buffer Pool**, which is: + +> A fixed-size array of memory frames + +--- + +## Core Structure (Conceptual) + +```rust +struct BufferPool { + frames: Vec, + page_table: HashMap, + policy: ReplacementPolicy, +} +``` + +--- + +## Conceptual View + +```text ++---------------------------------------------------+ +| Buffer Pool | ++---------------------------------------------------+ +| Frame 0 | Frame 1 | Frame 2 | ... | Frame N-1 | ++---------------------------------------------------+ +``` + +Each frame can hold **one page**. + +--- + +# 3. Frame Abstraction + +Each frame represents a **slot in memory**. + +--- + +## Conceptual Frame Layout + +```text ++----------------------------------+ +| Page Data (8 KB) | ++----------------------------------+ +| Metadata | +|----------------------------------| +| page_id | +| pin_count | +| dirty_flag | ++----------------------------------+ +``` + +--- + +## Key Properties + +- Fixed-size (aligned with page size: 8 KB) +- Contains: + - Page data + - Control metadata + +--- + +# 4. Page Table Mapping + +To locate pages quickly: + +```text +Page Table (HashMap) + +PageId → FrameId +``` + +--- + +## Diagram + +```text +Page Table: + +[Page 5] → Frame 2 +[Page 8] → Frame 7 +[Page 1] → Frame 0 +``` + +--- + +## Purpose + +- Avoid scanning buffer pool +- Enable O(1) lookup + +--- + +# 5. Buffer Pool Partitioning + +The buffer pool is **logically divided into two regions**: + +--- + +## 5.1 Reserved Region (System Pages) + +```text +[ RESERVED FRAMES ] +Frame 0 → Frame 127 +``` + +--- + +## 5.2 General Region (User Pages) + +```text +[ GENERAL FRAMES ] +Frame 127 → Frame (N-1) +``` + +--- + +## Full Layout + +```text ++-------------------------------------------------------+ +| RESERVED | RESERVED | ... | GENERAL | GENERAL | ... | +| Frames | Frames | | Frames | Frames | | ++-------------------------------------------------------+ + 0 ... 127 128 ... N-1 +``` + +--- + +# 6. Reservation Strategy + +## Key Design Decision + +A fixed number of frames are **reserved exclusively** for: + +- Catalog pages +- System metadata +- Frequently accessed internal structures + +--- + +## Constant Definition (Conceptual) + +```rust +const RESERVED_FRAMES: usize = 128; +``` + +--- + +## Why Reservation? + +### 1. Prevent System Page Eviction + +```text +Without reservation: +→ Catalog pages may get evicted +→ Leads to repeated disk reads +→ Performance degradation +``` + +--- + +### 2. Ensure Metadata Availability + +- Catalog is required for: + - Table lookup + - Schema resolution + +--- + +### 3. Isolation from User Workload + +```text +User queries → heavy page access +System pages → must remain stable +``` + +--- + +# 7. Reserved vs General Pool Behavior + +## Reserved Region + +```text +- Preloaded at startup +- Not part of general eviction policy +- Dedicated for catalog/system pages +- Have special eviction policy implemented + +``` + +--- + +## General Region + +```text +- Used for table data pages +- Managed by replacement policy +- Subject to eviction +``` + +--- + +## Diagram + +```text + BUFFER POOL + ++-------------------------------------------------------+ +| Reserved Zone | General Zone | +|------------------------|----------------------------------| +| Catalog Pages | Table Pages | +| System Metadata | User Data | +| Special Eviction | Eviction Enabled | ++-------------------------------------------------------+ +``` + +--- + +# 8. Access Flow Architecture + +## Data Page Fetch Flow + +```text +Request Page (page_id) + | + v + Check Page Table + / \ + / \ + Hit Miss + | | + v v + Return Select Victim + Page | + v + Replace Frame + | + v + Load Page +``` + +--- + +# 9. Victim Selection Scope + +Important Architectural Constraint + +```text +Victim selection ONLY applies to: + +→ General Region +→ Frames [RESERVED_FRAMES ... N-1] +``` + +--- + +## Diagram + +```text +Frames: + +[0 ... 127] → Reserved (excluded) +[128 ... N-1] → Eligible for eviction +``` + +--- + +# 10. Memory Layout Perspective + +## Logical Memory View + +```text +RAM +│ +├── Buffer Pool +│ ├── Reserved Frames (128) +│ │ ├── Catalog Page 0 +│ │ ├── Catalog Page 1 +│ │ └── ... +│ │ +│ └── General Frames +│ ├── Table Page A +│ ├── Table Page B +│ └── ... +│ +└── Other Runtime Memory +``` + +--- + +# 11. Preloading Mechanism (Architectural) + +At system startup: + +```text +1. Initialize buffer pool +2. Load catalog pages +3. Place them in reserved frames +``` + +--- + +## Diagram + +```text +Startup: + +Disk (catalog.json) + ↓ +Load Pages + ↓ +Store in Reserved Frames +``` + +--- + +# 12. Concurrency & Safety (Conceptual) + +Although not implementation-specific: + +- Frames are accessed via controlled interfaces +- Pin count ensures: + - No eviction during usage + +--- + +## Pinning Concept + +```text +pin_count > 0 → page in use → cannot evict +pin_count = 0 → eligible +``` + +--- + +# 13. Scalability Considerations + +## Fixed Size Design + +- Buffer pool size is static +- Trade-off: + - Predictable memory usage + - Limited flexibility + +--- + +## Partitioning Advantage + +```text +Reserved Zone → stability +General Zone → flexibility +``` + +--- + +# 14. Architectural Summary + +```text + BUFFER MANAGER + + +-----------------------------------+ + | Buffer Pool | + |-----------------------------------| + | Reserved Frames (System) | + |-----------------------------------| + | General Frames (User Data) | + |-----------------------------------| + | Replacement Policy | + |-----------------------------------| + | Page Table | + +-----------------------------------+ +``` + +--- + +# 16. Relation to RookDB Architecture + +The Buffer Manager: + +- Sits above the Page Layer +- Manages in-memory caching +- Reduces disk access overhead + + +--- From 7faf408e59ac708bd8997d71e911bdba7959c275 Mon Sep 17 00:00:00 2001 From: gopendra113 Date: Wed, 22 Apr 2026 03:15:48 +0530 Subject: [PATCH 4/5] updated the documentation --- .../projects/buffer-manager/api-reference.md | 346 ++++++++++++ .../projects/buffer-manager/architecture.md | 45 +- .../projects/buffer-manager/buffer-manager.md | 533 ++++++++++-------- .../buffer-manager/data-structures.md | 147 +++++ .../buffer-manager/replacement-policies.md | 460 +++++++++++++++ .../buffer-manager/replacement_policy.md | 402 ------------- 6 files changed, 1275 insertions(+), 658 deletions(-) create mode 100644 content/storage-engine/projects/buffer-manager/api-reference.md create mode 100644 content/storage-engine/projects/buffer-manager/data-structures.md create mode 100644 content/storage-engine/projects/buffer-manager/replacement-policies.md delete mode 100644 content/storage-engine/projects/buffer-manager/replacement_policy.md diff --git a/content/storage-engine/projects/buffer-manager/api-reference.md b/content/storage-engine/projects/buffer-manager/api-reference.md new file mode 100644 index 0000000..a90b6d0 --- /dev/null +++ b/content/storage-engine/projects/buffer-manager/api-reference.md @@ -0,0 +1,346 @@ +--- +title: Buffer Manager +sidebar_position: 3 +--- + +# Buffer Manager - API reference + + +The **Buffer Pool** is responsible for managing in-memory pages for multiple tables (files). It acts as an abstraction layer between disk storage and higher-level components such as the catalog and table manager. + +It supports: +- Multi-file page management +- Page caching (hit/miss handling) +- Replacement policies (LRU, Clock) +- Dirty page handling and flushing +- Reserved memory region for catalog pages + +--- + +# Core Concepts + +- **PageId** → uniquely identifies a page: + (table_name, page_number) + +- **Frames** → fixed-size memory slots storing pages + +- **Reserved Frames (0–127)** → used for catalog pages (never evicted) + +- **Data Frames (128+)** → used for table data (managed by replacement policy) + +--- + +# Exposed APIs + +--- + +## 1. fetch_page + +```rust +pub fn fetch_page( + &mut self, + table_name: String, + page_number: u32, +) -> io::Result<&mut Page> +``` + +### Description +Fetches a page from the buffer pool. If the page is not present, it is loaded from disk. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| table_name | String | Name of the table (file) | +| page_number | u32 | Page number to fetch | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(&mut Page) | Mutable reference to the page | +| Err(io::Error) | If page cannot be loaded | + +### Behavior + +1. Buffer Hit + - Page found in page_table + - Increments pin_count and usage_count + - Updates replacement policy (if not reserved) + +2. Buffer Miss + - Searches for free frame + - If none, evicts a victim frame + - Flushes dirty page if needed + +3. Disk Load + - Reads page from correct file using table_name + - Inserts into buffer and updates metadata + +### Notes +- Requires file to be registered in self.files +- Reserved frames are never evicted + +--- + +## 2. unpin_page + +```rust +pub fn unpin_page( + &mut self, + page_id: &PageId, + is_dirty: bool, +) -> io::Result<()> +``` + +### Description +Releases a previously pinned page and optionally marks it as dirty. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| page_id | &PageId | Page identifier | +| is_dirty | bool | Whether the page was modified | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | If page not found or already unpinned | + +### Behavior + +- Decrements pin_count +- Marks page as dirty if is_dirty = true + +### Notes +- Page cannot be evicted while pinned + +--- + +## 3. flush_page + +```rust +pub fn flush_page(&mut self, page_id: &PageId) -> io::Result<()> +``` + +### Description +Writes a dirty page from buffer to disk. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| page_id | &PageId | Page to flush | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | If page not found | + +### Behavior + +- Writes page to correct file using table_name +- Clears dirty flag +- Updates stats + +--- + +## 4. flush_all_pages + +```rust +pub fn flush_all_pages(&mut self) -> io::Result<()> +``` + +### Description +Flushes all dirty pages in the buffer pool to disk. + +### Inputs +None + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | On write failure | + +### Behavior + +- Iterates over all frames +- Writes all dirty pages to disk + +--- + +## 5. new_page + +```rust +pub fn new_page( + &mut self, + table_name: String, +) -> io::Result<(PageId, &mut Page)> +``` + +### Description +Creates a new page in a table file and loads it into the buffer. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| table_name | String | Target table | + +### Output + +| Return Type | Description | +|------------|-------------| +| (PageId, &mut Page) | New page identifier and reference | +| Err(io::Error) | If creation fails | + +### Behavior + +1. Calls create_page() on disk +2. Fetches new page into buffer +3. Marks page as dirty + +--- + +## 6. delete_page + +```rust +pub fn delete_page(&mut self, page_id: &PageId) -> io::Result<()> +``` + +### Description +Removes a page from the buffer pool. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| page_id | &PageId | Page to delete | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | If page is pinned | + +### Behavior + +- Removes mapping from page_table +- Clears frame metadata + +### Notes +- Does not delete from disk + +--- + +## 7. reset + +```rust +pub fn reset(&mut self) +``` + +### Description +Clears the entire buffer pool state. + +### Behavior + +- Clears all frames +- Clears page_table +- Clears files +- Resets statistics + +--- + +## 8. preload_database + +```rust +pub fn preload_database(&mut self, db_name: &str) -> io::Result<()> +``` + +### Description +Loads all table pages of a database into the buffer pool. + +### Inputs + +| Parameter | Type | Description | +|----------|------|-------------| +| db_name | &str | Database name | + +### Output + +| Return Type | Description | +|------------|-------------| +| Ok(()) | Success | +| Err(io::Error) | On failure | + +### Behavior + +- Resets buffer pool +- Iterates over all table files +- Loads pages starting from frame 128 +- Stops when buffer is full + +--- + +## 9. preload_catalog_pages + +```rust +pub fn preload_catalog_pages(&mut self) -> io::Result<()> +``` + +### Description +Loads the first two pages of each system catalog file into reserved frames. + +### Behavior + +- Opens catalog files: + - pg_database + - pg_table + - pg_column + - pg_constraint + - pg_index + - pg_type +- Loads pages 0 and 1 +- Stores them in reserved frames (0–127) +- Registers files in self.files + +### Notes + +- These pages are: + - Never evicted + - Not part of replacement policy + +--- + +# Summary + +The Buffer Pool now supports: + +- Multi-file page management +- Page-level caching +- Catalog + data separation +- Replacement policies +- Dirty page handling +- Preloading strategies + +--- + +# Example Usage + +```rust +let mut bp = BufferPool::new(Box::new(LRU::new())); + +bp.preload_catalog_pages()?; // load system catalogs +bp.preload_database("users")?; // load table data + +let page = bp.fetch_page("students".to_string(), 1)?; +bp.unpin_page(&PageId { table_name: "students".into(), page_number: 1 }, false)?; +``` \ No newline at end of file diff --git a/content/storage-engine/projects/buffer-manager/architecture.md b/content/storage-engine/projects/buffer-manager/architecture.md index f76cde2..2a2849c 100644 --- a/content/storage-engine/projects/buffer-manager/architecture.md +++ b/content/storage-engine/projects/buffer-manager/architecture.md @@ -43,13 +43,16 @@ The Buffer Manager is implemented as a **Buffer Pool**, which is: --- -## Core Structure (Conceptual) +## Core Structure ```rust struct BufferPool { - frames: Vec, - page_table: HashMap, - policy: ReplacementPolicy, + pub frames: Vec, + pub page_table: HashMap, + pub files: HashMap, + pub num_frames: usize, + pub policy: Box, + pub stats: BufferStats, } ``` @@ -81,11 +84,12 @@ Each frame represents a **slot in memory**. +----------------------------------+ | Page Data (8 KB) | +----------------------------------+ -| Metadata | -|----------------------------------| +| Frame Metadata : | | page_id | | pin_count | -| dirty_flag | +| dirty | +| usage_count | +| last_used | +----------------------------------+ ``` @@ -396,20 +400,19 @@ General Zone → flexibility # 14. Architectural Summary -```text - BUFFER MANAGER - - +-----------------------------------+ - | Buffer Pool | - |-----------------------------------| - | Reserved Frames (System) | - |-----------------------------------| - | General Frames (User Data) | - |-----------------------------------| - | Replacement Policy | - |-----------------------------------| - | Page Table | - +-----------------------------------+ +``` +┌─────────────────────────────────────────────────────┐ +│ Buffer Pool (Total) │ +│ 128 MB (configurable size) │ +├─────────────────────────────────────────────────────┤ +│ Reserved Region │ Data Region │ +│ (Frames 0-128) │ (Frames 129+) │ +│ [Catalog Pages] │ [Table Pages - Managed by Policy]│ +├─────────────────────────────────────────────────────┤ +│ Page | Page | Page | ... | Page | Page | ... | Page │ +├─────────────────────────────────────────────────────┤ +│ 8KB 8KB 8KB 8KB 8KB 8KB │ +└─────────────────────────────────────────────────────┘ ``` --- diff --git a/content/storage-engine/projects/buffer-manager/buffer-manager.md b/content/storage-engine/projects/buffer-manager/buffer-manager.md index 234a6ce..977d1e6 100644 --- a/content/storage-engine/projects/buffer-manager/buffer-manager.md +++ b/content/storage-engine/projects/buffer-manager/buffer-manager.md @@ -1,346 +1,409 @@ --- -title: Buffer Manager -sidebar_position: 3 +title: Buffer Manager Overview +sidebar_position: 1 --- -# Buffer Manager +# Buffer Manager - Complete Overview & Guide +## Introduction -The **Buffer Pool** is responsible for managing in-memory pages for multiple tables (files). It acts as an abstraction layer between disk storage and higher-level components such as the catalog and table manager. +The **Buffer Manager** is a critical component of RookDB's storage subsystem that manages in-memory caching of database pages. It sits between the execution layer and disk storage, significantly reducing I/O operations by maintaining a pool of frequently accessed pages in memory. -It supports: -- Multi-file page management -- Page caching (hit/miss handling) -- Replacement policies (LRU, Clock) -- Dirty page handling and flushing -- Reserved memory region for catalog pages +### What It Does + +- **Reduces Disk I/O**: Keeps frequently accessed pages in memory rather than repeatedly reading from disk +- **Manages Limited Memory**: Intelligently evicts pages when buffer is full using pluggable replacement policies +- **Ensures Correctness**: Tracks page modifications (dirty flag) and ensures they're written to disk +- **Multi-Table Support**: Handles pages from multiple database table files simultaneously +- **Measures Performance**: Tracks cache hits/misses to monitor buffer effectiveness --- -# Core Concepts +## Architecture at a Glance -- **PageId** → uniquely identifies a page: - (table_name, page_number) +``` +┌─────────────────────────────────────┐ +│ Query Layer / Execution Engine │ +└──────────────┬──────────────────────┘ + │ fetch_page / unpin_page + │ +┌──────────────▼──────────────────────┐ +│ Buffer Manager │ ◄─ This Component +│ (Caching + Replacement Policy) │ +└──────────────┬──────────────────────┘ + │ read_page / write_page + │ +┌──────────────▼──────────────────────┐ +│ Disk Manager │ +│ (Page Read/Write Operations) │ +└─────────────────────────────────────┘ +``` +### High-Level Architecture -- **Frames** → fixed-size memory slots storing pages +The Buffer Manager implements a **Buffer Pool**—a fixed-size array of memory frames where each frame holds one database page. -- **Reserved Frames (0–127)** → used for catalog pages (never evicted) +``` +┌─────────────────────────────────────────────────────┐ +│ Buffer Pool (Total) │ +│ 128 MB (configurable size) │ +├─────────────────────────────────────────────────────┤ +│ Reserved Region │ Data Region │ +│ (Frames 0-128) │ (Frames 129+) │ +│ [Catalog Pages] │ [Table Pages - Managed by Policy]│ +├─────────────────────────────────────────────────────┤ +│ Page | Page | Page | ... | Page | Page | ... | Page │ +├─────────────────────────────────────────────────────┤ +│ 8KB 8KB 8KB 8KB 8KB 8KB │ +└─────────────────────────────────────────────────────┘ +``` -- **Data Frames (128+)** → used for table data (managed by replacement policy) +### Key Concept: Pinning ---- +Pages are managed using a **pin count** mechanism: -# Exposed APIs +``` +fetch_page() → pin_count = 1 (page locked in buffer) +fetch_page() → pin_count = 2 (same page, multiple users) +unpin_page() → pin_count = 1 (one user done) +unpin_page() → pin_count = 0 (eligible for eviction) +``` ---- +Only frames with `pin_count = 0` can be evicted when space is needed. -## 1. fetch_page +--- -```rust -pub fn fetch_page( - &mut self, - table_name: String, - page_number: u32, -) -> io::Result<&mut Page> -``` +## Key Concepts -### Description -Fetches a page from the buffer pool. If the page is not present, it is loaded from disk. +| Concept | Explanation | +|---------|-------------| +| **PageId** | Unique identifier: (table_name, page_number) | +| **Frame** | Fixed-size memory slot (8 KB) holding one page | +| **Reserved Frames** | Frames 0-128 for catalog pages (never evicted) | +| **Data Frames** | Frames 129+ for table data (managed by replacement policy) | +| **Dirty Page** | Page that has been modified in memory but not flushed to disk | +| **Pin Count** | Number of active users holding a page; >0 means cannot evict | +| **Replacement Policy** | Algorithm that chooses which page to evict (Clock/LRU/LRU-K) | -### Inputs +--- -| Parameter | Type | Description | -|----------|------|-------------| -| table_name | String | Name of the table (file) | -| page_number | u32 | Page number to fetch | +## Documentation Structure -### Output +This folder contains specialized documentation for different aspects: -| Return Type | Description | -|------------|-------------| -| Ok(&mut Page) | Mutable reference to the page | -| Err(io::Error) | If page cannot be loaded | +### Core References -### Behavior +| Document | Contents | When to Read | +|----------|----------|--------------| +| **[API Reference](./api-reference.md)** | Complete method signatures and behavior | You need to call a specific method | +| **[Data Structures](./data-structures.md)** | Definition of PageId, FrameMetadata, BufferFrame, etc. | Understanding internal types | +| **[Architecture](./architecture.md)** | System design, frame layout, reservation strategy | How components fit together | +| **[Replacement Policies](./replacement_policies.md)** | Detailed Clock, LRU, and LRU-K algorithms | Choosing/tuning eviction strategy | -1. Buffer Hit - - Page found in page_table - - Increments pin_count and usage_count - - Updates replacement policy (if not reserved) +### Quick Navigation -2. Buffer Miss - - Searches for free frame - - If none, evicts a victim frame - - Flushes dirty page if needed +**I want to...** +- **Call a method** → See [API Reference](./api-reference.md) +- **Understand a data structure** → See [Data Structures](./data-structures.md) +- **Learn how it works internally** → See [Architecture](./architecture.md) +- **Choose a replacement policy** → See [Replacement Policies](./replacement-policies.md) +- **Get an Overview** → Keep reading this document -3. Disk Load - - Reads page from correct file using table_name - - Inserts into buffer and updates metadata +--- -### Notes -- Requires file to be registered in self.files -- Reserved frames are never evicted +## System Design Overview ---- +The buffer manager divides the buffer pool into two regions: -## 2. unpin_page +### Reserved Region (Frames 0-128) -```rust -pub fn unpin_page( - &mut self, - page_id: &PageId, - is_dirty: bool, -) -> io::Result<()> +``` +┌────────────────────────────────┐ +│ RESERVED FRAMES (0-128) │ +│ System Catalog Pages │ +├────────────────────────────────┤ +│ pg_database | pg_table │ +│ pg_column | pg_constraint │ +│ pg_index | pg_type │ +└────────────────────────────────┘ ``` -### Description -Releases a previously pinned page and optionally marks it as dirty. +**Purpose**: Store system catalog metadata that must always be accessible +- Catalog is required for every query (table lookups, schema checks) +- Never evicted, always available +- Takes up ~1 MB of the buffer -### Inputs +### Data Region (Frames 129+) -| Parameter | Type | Description | -|----------|------|-------------| -| page_id | &PageId | Page identifier | -| is_dirty | bool | Whether the page was modified | +``` +┌────────────────────────────────┐ +│ DATA FRAMES (129+) │ +│ User Table Pages │ +├────────────────────────────────┤ +│ Table Data Pages (Managed by │ +│ Replacement Policy) │ +└────────────────────────────────┘ +``` -### Output +**Purpose**: Cache actual table data +- Subject to eviction when new pages needed +- Replacement policy decides which pages to evict +- Takes up ~127 MB of the buffer -| Return Type | Description | -|------------|-------------| -| Ok(()) | Success | -| Err(io::Error) | If page not found or already unpinned | +--- -### Behavior +## Typical Workflow -- Decrements pin_count -- Marks page as dirty if is_dirty = true +``` +1. INITIALIZATION + ├─ Create BufferPool with chosen replacement policy + └─ Register all table files: buffer.register_file("users")? + +2. DURING QUERY EXECUTION + ├─ fetch_page("users", 0)? ← Get page from buffer + ├─ [Buffer Hit] ← Already in memory + ├─ OR + ├─ [Buffer Miss] ← Read from disk + │ ├─ Find free frame (or evict if needed) + │ └─ Read from disk, place in frame + │ + ├─ Modify page data + └─ unpin_page(&page_id, is_dirty)? ← Release page + +3. ON SHUTDOWN + └─ flush_all()? ← Write all dirty pages to disk -### Notes -- Page cannot be evicted while pinned +``` --- -## 3. flush_page +## Configuration + +The buffer manager is configured via constants in `mod.rs`: ```rust -pub fn flush_page(&mut self, page_id: &PageId) -> io::Result<()> +pub const PAGE_SIZE: usize = 8192; // 8 KB per page +pub const BUFFER_SIZE: usize = 128 * 1024 * 1024; // 128 MB total +pub const RESERVED_FRAMES: usize = 129; // Frames 0-128 reserved ``` -### Description -Writes a dirty page from buffer to disk. - -### Inputs - -| Parameter | Type | Description | -|----------|------|-------------| -| page_id | &PageId | Page to flush | - -### Output +### Implications -| Return Type | Description | -|------------|-------------| -| Ok(()) | Success | -| Err(io::Error) | If page not found | - -### Behavior +``` +Total Frames = 128 MB / 8 KB = 16,384 frames +Reserved = 129 frames (~1 MB) +Data Frames = 16,255 frames (~127 MB) +``` -- Writes page to correct file using table_name -- Clears dirty flag -- Updates stats +To adjust for your workload: +- **More hot data?** Increase `BUFFER_SIZE` +- **More concurrent queries?** May need larger buffer for pin count headroom +- **Smaller buffer?** Decrease `BUFFER_SIZE` (minimum should fit catalog + working set) --- -## 4. flush_all_pages - -```rust -pub fn flush_all_pages(&mut self) -> io::Result<()> -``` +## Choosing a Replacement Policy -### Description -Flushes all dirty pages in the buffer pool to disk. +Three policies are available. Pick based on your workload: -### Inputs -None +### Clock Policy +- **Best for**: General workloads, sequential access +- **Memory overhead**: Minimal +- **Speed**: Very fast +- **When**: Unsure, or memory-constrained -### Output +### LRU Policy +- **Best for**: Working set fits in buffer, strong temporal locality +- **Memory overhead**: Moderate (timestamps per frame) +- **Speed**: Medium +- **When**: Good hit ratios expected, memory available -| Return Type | Description | -|------------|-------------| -| Ok(()) | Success | -| Err(io::Error) | On write failure | +### LRU-K Policy +- **Best for**: Mixed hot/cold access patterns, cache pollution resistance +- **Memory overhead**: High (K timestamps per frame) +- **Speed**: Slower +- **When**: Need sophisticated eviction behavior -### Behavior - -- Iterates over all frames -- Writes all dirty pages to disk +See [Replacement Policies](./replacement-policies.md) for detailed comparison. --- -## 5. new_page - -```rust -pub fn new_page( - &mut self, - table_name: String, -) -> io::Result<(PageId, &mut Page)> -``` +## Integration Points -### Description -Creates a new page in a table file and loads it into the buffer. +### Reading Pages -### Inputs +```rust +// Fetch a page from the buffer (may load from disk) +let page = buffer.fetch_page("users".to_string(), 0)?; -| Parameter | Type | Description | -|----------|------|-------------| -| table_name | String | Target table | +// Use the page -### Output +// Release the page +buffer.unpin_page(&page_id, false)?; // Not modified +``` -| Return Type | Description | -|------------|-------------| -| (PageId, &mut Page) | New page identifier and reference | -| Err(io::Error) | If creation fails | +### Writing Pages -### Behavior +```rust +// Fetch page +let mut page = buffer.fetch_page("users".to_string(), 0)?; -1. Calls create_page() on disk -2. Fetches new page into buffer -3. Marks page as dirty +// Modify page ---- +// Release as dirty +buffer.unpin_page(&page_id, true)?; // Mark for flush +``` -## 6. delete_page +### Creating Pages ```rust -pub fn delete_page(&mut self, page_id: &PageId) -> io::Result<()> -``` +// Create a new page in a table +let (page_id, page) = buffer.new_page("users".to_string())?; -### Description -Removes a page from the buffer pool. +// Populate page +// ... -### Inputs +// Mark as dirty (will be flushed) +buffer.unpin_page(&page_id, true)?; +``` -| Parameter | Type | Description | -|----------|------|-------------| -| page_id | &PageId | Page to delete | +--- -### Output +## Error Handling Guide -| Return Type | Description | -|------------|-------------| -| Ok(()) | Success | -| Err(io::Error) | If page is pinned | +### Common Errors and Fixes -### Behavior +```rust -- Removes mapping from page_table -- Clears frame metadata +// Error: "All frames are pinned" +// → Cause: Pin count leak (fetch without unpin) +// → Fix: Ensure every fetch_page has matching unpin_page -### Notes -- Does not delete from disk +// Error: Double unpin +// → Cause: unpin called twice on same page +// → Fix: Track pin count, unpin only once per fetch +``` ---- +### Pin Count Correctness -## 7. reset +**Critical**: Every `fetch_page()` must have matching `unpin_page()`: ```rust -pub fn reset(&mut self) +// CORRECT +let page = buffer.fetch_page("users", 0)?; // +1 +buffer.unpin_page(&page_id, true)?; // -1 + +// INCORRECT (Leak) +let page = buffer.fetch_page("users", 0)?; // +1 +// ... forgot to unpin ... +// page stays pinned forever! + +// INCORRECT (Double fetch) +let page = buffer.fetch_page("users", 0)?; // +1 +let page2 = buffer.fetch_page("users", 0)?; // +2 (same page) +buffer.unpin_page(&page_id, true)?; // -1 (still pinned!) ``` -### Description -Clears the entire buffer pool state. +--- -### Behavior +## Performance Monitoring -- Clears all frames -- Clears page_table -- Clears files -- Resets statistics +### Key Metrics ---- +```rust +// Get statistics +let stats = &buffer.stats; -## 8. preload_database +// Hit ratio (0.0 to 1.0, higher is better) +let hit_ratio = stats.hit_ratio(); -```rust -pub fn preload_database(&mut self, db_name: &str) -> io::Result<()> +// Counts +println!("Hits: {}", stats.hit_count); +println!("Misses: {}", stats.miss_count); +println!("Evictions: {}", stats.eviction_count); +println!("Dirty flushes: {}", stats.dirty_flush_count); ``` -### Description -Loads all table pages of a database into the buffer pool. +### Healthy vs Unhealthy -### Inputs +| Metric | Healthy | Problem | +|--------|---------|---------| +| Hit Ratio | > 80% | < 50% (buffer too small?) | +| Evictions | Proportional to workload | Very high (working set > buffer?) | +| Dirty Flushes | Matches write operations | Unexpected patterns? | -| Parameter | Type | Description | -|----------|------|-------------| -| db_name | &str | Database name | +### Tuning Based on Metrics -### Output - -| Return Type | Description | -|------------|-------------| -| Ok(()) | Success | -| Err(io::Error) | On failure | +``` +IF hit_ratio < 50%: + → Increase BUFFER_SIZE + → Try LRU instead of Clock + → Check if working set fits + +IF eviction_count is very high: + → Increase BUFFER_SIZE + → Try LRU-K for better selectivity + +IF pin_count errors: + → Check for fetch/unpin mismatches + → Reduce concurrent queries + → Increase BUFFER_SIZE for headroom +``` -### Behavior +--- -- Resets buffer pool -- Iterates over all table files -- Loads pages starting from frame 128 -- Stops when buffer is full +## Dirty Page Management ---- +Pages are marked dirty when modified and flushed to disk during eviction or explicit flush: -## 9. preload_catalog_pages +### Marking Pages ```rust -pub fn preload_catalog_pages(&mut self) -> io::Result<()> +// Mark page as modified +buffer.unpin_page(&page_id, true)?; // is_dirty = true ``` -### Description -Loads the first two pages of each system catalog file into reserved frames. +### Automatic Flushing -### Behavior - -- Opens catalog files: - - pg_database - - pg_table - - pg_column - - pg_constraint - - pg_index - - pg_type -- Loads pages 0 and 1 -- Stores them in reserved frames (0–127) -- Registers files in self.files - -### Notes - -- These pages are: - - Never evicted - - Not part of replacement policy +``` +When evicting a frame: + IF frame.metadata.dirty: + → Write page to disk + → Clear dirty flag + → Update statistics + THEN: + → Reuse frame for new page +``` ---- +### Explicit Flushing -# Summary +```rust +// Flush single page +buffer.flush_page(&page_id)?; -The Buffer Pool now supports: +// Flush all dirty pages +buffer.flush_all()?; +``` -- Multi-file page management -- Page-level caching -- Catalog + data separation -- Replacement policies -- Dirty page handling -- Preloading strategies +**When to flush explicitly**: +- Before shutdown (ensure durability) +- After transaction commit +- Before checkpoints +- Before backup operations --- -# Example Usage - -```rust -let mut bp = BufferPool::new(Box::new(LRU::new())); - -bp.preload_catalog_pages()?; // load system catalogs -bp.preload_database("users")?; // load table data - -let page = bp.fetch_page("students".to_string(), 1)?; -bp.unpin_page(&PageId { table_name: "students".into(), page_number: 1 }, false)?; -``` \ No newline at end of file +## Summary + +The Buffer Manager: +- Caches pages in memory to reduce disk I/O +- Manages memory via pluggable replacement policies +- Handles multiple table files +- Ensures data durability with dirty tracking +- Provides pin-based concurrency control +- Tracks performance with comprehensive statistics + +Use it correctly by: +- Pinning/unpinning properly +- Marking modifications +- Choosing appropriate policies +- Monitoring performance diff --git a/content/storage-engine/projects/buffer-manager/data-structures.md b/content/storage-engine/projects/buffer-manager/data-structures.md new file mode 100644 index 0000000..e72c7d4 --- /dev/null +++ b/content/storage-engine/projects/buffer-manager/data-structures.md @@ -0,0 +1,147 @@ +# Buffer Manager Data Structures + +This document describes the key data structures used in the Buffer Manager component of RookDB. + +## PageId + +Represents a unique identifier for a page in the database. + +```rust +pub struct PageId { + pub table_name: String, + pub page_number: u32, +} +``` + +- `table_name`: The name of the table this page belongs to. +- `page_number`: The page number within the table. + +## FrameMetadata + +Contains metadata associated with a buffer frame. + +```rust +pub struct FrameMetadata { + pub page_id: Option, // which page currently resides in this frame + pub dirty: bool, // whether page modified + pub pin_count: u32, // number of active users + pub usage_count: u32, // used by clock policy + pub last_used: u64, // timestamp for LRU +} +``` + +- `page_id`: The ID of the page currently stored in this frame, or `None` if empty. +- `dirty`: Indicates if the page has been modified since loading. +- `pin_count`: Number of active users currently accessing this frame. +- `usage_count`: Used by the Clock replacement policy to track usage. +- `last_used`: Timestamp of the last access, used by LRU policies. + +## BufferFrame + +Represents a single frame in the buffer pool, containing a page and its metadata. + +```rust +pub struct BufferFrame { + pub page: Page, + pub metadata: FrameMetadata, +} +``` + +- `page`: The actual page data stored in memory. +- `metadata`: Metadata associated with this frame. + +## BufferStats + +Tracks statistics for buffer pool performance. + +```rust +pub struct BufferStats { + pub hit_count: u64, + pub miss_count: u64, + pub eviction_count: u64, + pub dirty_flush_count: u64, +} +``` + +- `hit_count`: Number of times a requested page was found in the buffer. +- `miss_count`: Number of times a requested page was not in the buffer. +- `eviction_count`: Number of times a page was evicted from the buffer. +- `dirty_flush_count`: Number of times a dirty page was written to disk. + +## BufferPool + +The main buffer pool structure that manages frames, page mapping, and replacement policies. + +```rust +pub struct BufferPool { + pub frames: Vec, + pub page_table: HashMap, + pub files: HashMap, // MULTI-FILE SUPPORT + pub num_frames: usize, + pub policy: Box, + pub stats: BufferStats, +} +``` + +- `frames`: Vector of buffer frames containing the actual page data. +- `page_table`: Maps page IDs to frame indices for quick lookup. +- `files`: Maps file names to open file handles for multi-file support. +- `num_frames`: Total number of frames in the buffer pool. +- `policy`: The replacement policy used for eviction decisions. +- `stats`: Statistics tracking buffer performance. + +## ReplacementPolicy Trait + +Defines the interface for page replacement policies. + +```rust +pub trait ReplacementPolicy { + // Select a victim frame for eviction + fn victim(&mut self, frames: &mut Vec) -> Option; + + // Called whenever a frame is accessed + fn record_access(&mut self, frame_id: usize); +} +``` + +## ClockPolicy + +Implements the Clock (Second Chance) page replacement algorithm. + +```rust +pub struct ClockPolicy { + pub hand: usize, +} +``` + +- `hand`: Current position of the clock hand in the frame list. + +## LRUPolicy + +Implements the Least Recently Used (LRU) page replacement algorithm. + +```rust +pub struct LRUPolicy { + timestamps: HashMap, + current_time: u64, +} +``` + +- `timestamps`: Maps frame IDs to their last access timestamps. +- `current_time`: Global timestamp counter for access ordering. + +## LRUKPolicy + +Implements the LRU-K page replacement algorithm, which considers the K most recent accesses. + +```rust +pub struct LRUKPolicy { + k: usize, + current_time: u64, + history: HashMap>, // frame_id -> access timestamps +} +``` + +- `k`: The number of recent accesses to consider. +- `current_time`: Global timestamp counter. +- `history`: Maps frame IDs to vectors of their recent access timestamps. diff --git a/content/storage-engine/projects/buffer-manager/replacement-policies.md b/content/storage-engine/projects/buffer-manager/replacement-policies.md new file mode 100644 index 0000000..c83d81d --- /dev/null +++ b/content/storage-engine/projects/buffer-manager/replacement-policies.md @@ -0,0 +1,460 @@ +# Buffer Manager Replacement Policies + +## Overview + +The Buffer Manager in RookDB is responsible for managing the buffer pool, which caches database pages in memory to reduce disk I/O operations. A critical component of the buffer manager is the **replacement policy**, which determines which page to evict from the buffer when space is needed for a new page. This documentation provides a detailed explanation of the replacement policies implemented in RookDB, including their algorithms, code implementations, and integration with the buffer pool. + +The buffer manager supports three replacement policies: +1. **Clock Policy** - A simple, efficient approximation of LRU. +2. **LRU (Least Recently Used) Policy** - Evicts the least recently accessed page. +3. **LRU-K Policy** - An advanced policy that considers the history of the last K accesses. + +--- + +## 1. Replacement Policy Trait + +All replacement policies implement the `ReplacementPolicy` trait, which defines the interface for selecting victims and recording accesses. + +### Trait Definition + +```rust +pub trait ReplacementPolicy { + // Select a victim frame for eviction + fn victim(&mut self, frames: &mut Vec) -> Option; + + // Called whenever a frame is accessed + fn record_access(&mut self, frame_id: usize); +} +``` + +### Trait Methods Explained + +| Method | Purpose | Returns | +|--------|---------|---------| +| `victim(&mut self, frames: &mut Vec)` | Selects a frame index to evict from the buffer. Must skip pinned frames (pin_count > 0). Returns None if no victim can be found. | `Option` - Frame index or None | +| `record_access(&mut self, frame_id: usize)` | Called whenever a frame is accessed (buffer hit or page load). Updates the policy's internal state to track access patterns. | `()` - No return value | + +### Key Properties + +- **Frame Independence**: Policies operate on frame indices relative to `RESERVED_FRAMES` (system frames). +- **Pin Count Awareness**: All policies must respect pin counts and never evict pinned frames. +- **Stateful**: Each policy maintains internal state to track access patterns. +- **Trait Objects**: Used as `Box` in BufferPool for runtime polymorphism. + +--- + +## 2. Clock Replacement Policy + +### Algorithm Overview + +The Clock policy is a low-overhead, circular sweep algorithm that approximates LRU behavior. It uses a clock hand pointer that rotates through frames, giving each frame a "second chance" before eviction. + +**Key Concept:** +- Each frame has a `usage_count` (0 or 1 bit). +- A clock hand sweeps through frames in circular order. +- When a frame is accessed, its `usage_count` is set to 1. +- On eviction, frames with `usage_count == 0` are selected first. +- Frames with `usage_count == 1` get a second chance (set to 0) and the hand continues. + +### How it Works: Step by Step + +1. **Page Access** → `usage_count` set to 1 in BufferPool +2. **Need Eviction** → Clock policy victim selection begins +3. **Clock Hand Sweep** → Examine current frame at hand position +4. **Check Frame State**: + - If **pinned** → Skip to next frame + - If **not pinned AND usage_count == 0** → Evict this frame + - If **not pinned AND usage_count == 1** → Give second chance (set to 0), move hand +5. **Hand Movement** → `hand = (hand + 1) % num_frames` +6. **Termination** → After scanning 2 × buffer_size frames, return None (all pinned) + +### Advantages + +- **Minimal Memory Overhead**: Only one bit per frame (usage_count) +- **Fast Access Recording**: O(1) operation +- **Scalable**: Works well with large buffers +- **Cache-Friendly**: Sequential access pattern +- **Good Practical Performance**: Acceptable hit ratio + +### Disadvantages + +- **Less Accurate than LRU**: May evict recently used pages +- **Correlated Accesses**: Can cause thrashing with certain patterns +- **Second Chance Bias**: Heavily accessed pages get repeated chances + +### Code Implementation + +```rust +pub struct ClockPolicy { + pub hand: usize, // Current position in the circular buffer +} + +impl ClockPolicy { + /// Create a new Clock policy with hand at position 0 + pub fn new() -> Self { + Self { hand: 0 } + } +} + +impl ReplacementPolicy for ClockPolicy { + + fn victim(&mut self, frames: &mut Vec) -> Option { + let n = frames.len(); + let mut scanned = 0; + + // Scan up to 2 full rotations + while scanned < 2 * n { + let frame = &mut frames[self.hand]; + + // Step 1: Skip pinned frames (actively in use) + if frame.metadata.pin_count == 0 { + + // Step 2: If usage_count is 0, this frame is a victim + if frame.metadata.usage_count == 0 { + let victim = self.hand; + self.hand = (self.hand + 1) % n; // Advance for next eviction + return Some(victim); + } else { + // Step 3: Give second chance - clear the usage bit + frame.metadata.usage_count = 0; + } + } + + // Step 4: Move hand to next frame + self.hand = (self.hand + 1) % n; + scanned += 1; + } + + // All frames are pinned + None + } + + fn record_access(&mut self, _frame_id: usize) { + // NO-OP: Access recording is handled directly in BufferPool::fetch_page + // BufferPool sets frame.metadata.usage_count = 1 on every access + } +} +``` +### Implementation Details + +| Detail | Explanation | +|-----------------------------|--------------------------------------------------| +| `hand: usize` | Circular pointer to current frame being examined | +| `scanned < 2 * n` | Allow up to 2 complete rotations before giving up | +| `pin_count == 0` | Only consider unpinned frames for eviction | +| `usage_count == 0` | Immediate victim candidate | +| `usage_count = 0` | Second chance mechanism | +| `hand = (hand + 1) % n` | Circular wraparound | +| `record_access` | No-op; usage tracking in `BufferPool` | + + +--- + +## 3. LRU (Least Recently Used) Replacement Policy + +### Algorithm Overview + +LRU evicts the page that has not been accessed for the longest time. It maintains a logical timestamp for each frame, updating it on every access. When eviction is needed, the frame with the smallest (oldest) timestamp is selected. + +**Key Concept:** +- Each frame gets a timestamp on every access +- Global `current_time` counter increments on each access +- Victim = frame with minimum timestamp +- Skip pinned frames + +### How it Works: Step by Step + +1. **Page Access** → `current_time++`, `timestamps[frame_id] = current_time` +2. **Need Eviction** → Scan all frames to find minimum timestamp +3. **Find Victim**: + - For each **unpinned** frame: + - Get its timestamp (0 if never accessed) + - Track the minimum timestamp seen + - Select frame with minimum timestamp +4. **Return** → Frame index with oldest access time + +### Advantages + +- **Optimal for Sequential Workloads**: Excellent locality exploitation +- **Simple and Intuitive**: Easy to understand and reason about +- **Good General Performance**: Works well for most workloads +- **Predictable**: Deterministic behavior based on access history +- **Industry Standard**: Widely used in real databases + +### Disadvantages + +- **Higher Memory Overhead**: One u64 timestamp per frame (~8 bytes) +- **O(n) Victim Selection**: Linear scan through all frames +- **Sequential Flooding**: Vulnerable to full-scan access patterns +- **No Distinction**: All accesses weighted equally regardless of pattern + +### Code Implementation + +```rust +pub struct LRUPolicy { + timestamps: HashMap, // frame_id -> last access time + current_time: u64, // global logical clock +} + +impl LRUPolicy { + /// Create a new LRU policy + pub fn new() -> Self { + Self { + timestamps: HashMap::new(), + current_time: 0, + } + } +} + +impl ReplacementPolicy for LRUPolicy { + + fn victim(&mut self, frames: &mut Vec) -> Option { + let mut victim_index = None; + let mut oldest_time = u64::MAX; + + // Scan all frames to find the least recently used + for (i, frame) in frames.iter().enumerate() { + + // Step 1: Skip pinned frames + if frame.metadata.pin_count != 0 { + continue; + } + + // Step 2: Get timestamp for this frame (0 if never accessed) + let time = *self.timestamps.get(&i).unwrap_or(&0); + + // Step 3: Track the minimum timestamp + if time < oldest_time { + oldest_time = time; + victim_index = Some(i); + } + } + + // Return the frame with the oldest timestamp + victim_index + } + + fn record_access(&mut self, frame_id: usize) { + // Step 1: Increment global time + self.current_time += 1; + + // Step 2: Record this frame's access with the new timestamp + self.timestamps.insert(frame_id, self.current_time); + } +} +``` +### Implementation Details + +| Detail | Explanation | +|--------------------------------|----------------------------------------------------------| +| `timestamps: HashMap` | Maps `frame_id` to last access timestamp | +| `current_time: u64` | Global logical clock, increments on each access | +| `u64::MAX` | Used as initial "oldest_time" for comparison | +| `unwrap_or(&0)` | Frames never accessed have timestamp `0` (evicted first) | +| Linear scan | O(n) where n = number of frames | +| HashMap insert | Constant-time timestamp update | +--- + +## 4. LRU-K Replacement Policy + +### Algorithm Overview + +LRU-K is an advanced policy that considers the history of the last K accesses. Instead of just the last access time, it tracks the K most recent accesses and uses the "backward K-distance" metric for eviction decisions. + +**Key Concepts:** +- **Backward K-Distance**: For a frame with ≥ K accesses, it's the time since the K-th most recent access +- **Infinite Distance**: Frames with < K accesses get distance = ∞ (low eviction priority) +- **Access History**: Each frame maintains a vector of its last K access timestamps +- **FIFO Overflow**: When history exceeds K entries, the oldest is removed + +--- +### Why LRU-K Matters + +**Problem with LRU:** +- Sequential scan of 1000 pages → all get recent timestamps +- These displace cached hot data despite being accessed only once! + +**Solution with LRU-K:** +- Hot page: accessed K times (backward K-distance = large) +- Sequential page: accessed once (backward K-distance = 0 or not counted) +- **Result:** Hot page is protected! + +--- + +### How it Works: Step by Step + +1. **Page Access** → Add timestamp to frame's history, keep only last K +2. **Calculate Distance** → `current_time - history[0]` (if K accesses exist) +3. **Handle New Pages** → Frames with `< K` accesses get distance = ∞ +4. **Select Victim** → Frame with **MAXIMUM** backward K-distance +5. **Eviction** → Evict the least "K-used" page + +--- +### Advantages + +- **Sequential Flooding Resistant**: Protects repeatedly accessed pages +- **Intelligent History Tracking**: Considers patterns, not just recency +- **Configurable**: K parameter tunes sensitivity +- **Workload Adaptive**: Handles mixed access patterns well +- **Cache Pollution Prevention**: Doesn't evict hot data for sequential scans + +### Disadvantages + +- **Highest Memory Overhead**: K timestamps per frame (K × 8 bytes) +- **Complex Implementation**: More code, harder to debug +- **Parameter Tuning**: K value must be chosen for workload +- **Vector Operations**: Removing oldest timestamp is O(K) +- **O(n) Victim Selection**: Still must scan all frames + + +### Code Implementation + +```rust +pub struct LRUKPolicy { + k: usize, // Number of accesses to track + current_time: u64, // Global logical clock + history: HashMap>, // frame_id -> last K access times +} + +impl LRUKPolicy { + /// Create a new LRU-K policy with K accesses to track + pub fn new(k: usize) -> Self { + Self { + k, + current_time: 0, + history: HashMap::new(), + } + } + + /// Calculate backward K-distance for a frame + fn backward_k_distance(&self, frame_id: usize) -> u64 { + match self.history.get(&frame_id) { + Some(timestamps) => { + if timestamps.len() < self.k { + u64::MAX + } else { + let kth_time = timestamps[0]; + self.current_time - kth_time + } + } + None => u64::MAX, + } + } +} + +impl ReplacementPolicy for LRUKPolicy { + + fn victim(&mut self, frames: &mut Vec) -> Option { + let mut victim_index = None; + let mut max_distance = 0; + + for (i, frame) in frames.iter().enumerate() { + if frame.metadata.pin_count != 0 { + continue; + } + + let distance = self.backward_k_distance(i); + if victim_index.is_none() || distance > max_distance { + max_distance = distance; + victim_index = Some(i); + } + } + + victim_index + } + + fn record_access(&mut self, frame_id: usize) { + self.current_time += 1; + + let entry = self.history.entry(frame_id).or_insert(Vec::new()); + entry.push(self.current_time); + + if entry.len() > self.k { + entry.remove(0); + } + } +} +``` +### Implementation Details + +Note : `K` is hardcoded to 3 in the implementation. + +| Detail | Explanation | +|--------------------------------------|--------------------------------------------------------------| +| `k: usize` | Number of recent accesses to track per frame | +| `history: HashMap>` | Maps `frame_id` to vector of last K timestamps | +| `current_time: u64` | Global logical clock | +| `u64::MAX` | Used to represent infinite distance (`< K` accesses) | +| `history[0]` | Oldest timestamp among the K most recent (kth access) | +| `entry.remove(0)` | Remove oldest when history exceeds K (O(K) cost) | +| Maximum selection | Unlike LRU (minimum), we select MAXIMUM distance | + +--- + +## 5. Integration with Buffer Pool + +### Buffer Pool Architecture + +The `BufferPool` struct uses a replacement policy to manage page evictions when no free frames are available. + +```rust +pub struct BufferPool { + pub frames: Vec, + pub page_table: HashMap, + pub files: HashMap, + pub num_frames: usize, + pub policy: Box, + pub stats: BufferStats, +} +``` + +The `BufferPool` uses the replacement policy through the `policy` field, which is a `Box`. + +### Key Integration Points + +1. **Policy Selection:** The buffer pool is initialized with a chosen policy: + +```rust +pub fn new(policy: Box) -> Self { ... } +``` + +2. **Access Recording:** In `fetch_page`, after finding or loading a page: + +```rust +if frame_index >= RESERVED_FRAMES { + self.policy.record_access(frame_index - RESERVED_FRAMES); +} +``` + +> **Note:** Only non-reserved frames are managed by the policy. + +3. **Victim Selection:** In `fetch_page`, when no free frame is available: + +```rust +let victim = self.policy.victim(&mut self.frames); +``` + +4. **Frame Indexing:** Policies work with frame indices relative to `RESERVED_FRAMES`. +The buffer has reserved frames `(0 to RESERVED_FRAMES-1)` for system pages, and the rest are managed by the policy. +--- + +## 6. Summary Table + +### Replacement Policies At a Glance + +| Aspect | Clock | LRU | LRU-K | +|--------|-------|-----|-------| +| **Algorithm** | Circular sweep | Min timestamp | Max K-distance | +| **Victim Selection** | O(1-2n) | O(n) | O(n) | +| **Access Record** | O(1) | O(1) | O(K) | +| **Implementation** | Simple | Medium | Complex | +| **Sequential Immunity** | No | No | Yes | + +--- + +## Conclusion + +The Buffer Manager's replacement policies provide a spectrum of options from simple and efficient to sophisticated and adaptive. The choice of policy significantly impacts database performance. Clock offers minimal overhead, LRU provides excellent general-purpose performance, and LRU-K protects against cache pollution in scan-heavy workloads. + +The modular design with the `ReplacementPolicy` trait allows RookDB to remain flexible, enabling easy policy switching and future extensions without modifying the core buffer pool logic. + diff --git a/content/storage-engine/projects/buffer-manager/replacement_policy.md b/content/storage-engine/projects/buffer-manager/replacement_policy.md deleted file mode 100644 index 018dd71..0000000 --- a/content/storage-engine/projects/buffer-manager/replacement_policy.md +++ /dev/null @@ -1,402 +0,0 @@ -# Buffer Replacement Policies (Implementation-Level Explanation) - -This document explains the **actual implementation** of buffer replacement policies in RookDB, -including **code-level behavior** from: - -- `policy.rs` -- `lru.rs` -- `clock.rs` -- `lru_k.rs` - -The buffer manager relies on these policies to select a **victim frame** when the buffer pool is full. - ---- - -# 1. Policy Trait (policy.rs) - -All replacement policies implement a common interface. - -```rust -pub trait ReplacementPolicy { - fn record_access(&mut self, frame_id: usize); - fn victim(&mut self, frames: &mut [Frame]) -> Option; -} -``` - -## Explanation - -### `record_access(frame_id)` -- Called whenever a page is: - - Fetched - - Hit in buffer -- Updates internal metadata of the policy - -### `victim(frames)` -- Selects a frame for eviction -- Must: - - Skip pinned frames (`pin_count > 0`) - - Return `None` if no frame is evictable - ---- - -# 2. Frame Interaction (Important) - -All policies operate on `Frame`. - -Typical structure: - -```rust -pub struct Frame { - pub page_id: Option, - pub pin_count: u32, - pub is_dirty: bool, -} -``` - -## Eviction Rules - -```text -pin_count > 0 → cannot evict -pin_count == 0 → eligible -``` - ---- - -# 3. LRU Policy (lru.rs) - -## Core Idea - -Evict the **least recently used frame**. - ---- - -## Internal State (Typical) - -```rust -pub struct LRU { - pub order: Vec, // stores frame_ids -} -``` - -- Front → most recently used -- Back → least recently used - ---- - -## record_access() - -```rust -fn record_access(&mut self, frame_id: usize) { - self.order.retain(|&id| id != frame_id); - self.order.insert(0, frame_id); -} -``` - -### Explanation - -1. Remove frame if already present -2. Insert at front (most recent) - ---- - -## victim() - -```rust -fn victim(&mut self, frames: &mut [Frame]) -> Option { - for &frame_id in self.order.iter().rev() { - if frames[frame_id].pin_count == 0 { - return Some(frame_id); - } - } - None -} -``` - -### Explanation - -- Traverse from **least recent → most recent** -- Return first unpinned frame - ---- - -## Diagram - -```text -MRU → [2, 5, 1, 7, 3] ← LRU - -Eviction scan → -3 → if unpinned → victim -``` - ---- - -# 4. Clock Policy (clock.rs) - -## Core Idea - -Efficient approximation of LRU using: -- Circular pointer -- Reference bit - ---- - -## Internal State - -```rust -pub struct Clock { - pub hand: usize, - pub ref_bits: Vec, -} -``` - ---- - -## record_access() - -```rust -fn record_access(&mut self, frame_id: usize) { - self.ref_bits[frame_id] = true; -} -``` - ---- - -## victim() - -```rust -fn victim(&mut self, frames: &mut [Frame]) -> Option { - let n = frames.len(); - - for _ in 0..(2 * n) { - let i = self.hand; - - if frames[i].pin_count == 0 { - if !self.ref_bits[i] { - self.hand = (self.hand + 1) % n; - return Some(i); - } else { - self.ref_bits[i] = false; - } - } - - self.hand = (self.hand + 1) % n; - } - - None -} -``` - ---- - -## Explanation - -1. Check frame at `hand` -2. If: - - `ref_bit = 1` → give second chance → set to 0 - - `ref_bit = 0` → evict -3. Move pointer circularly - ---- - -## Diagram - -```text -Frames: [0] [1] [2] [3] -Ref bits: 1 0 1 0 - ↑ - hand - -Step: -0 → reset -1 → evict -``` - ---- - -# 5. LRU-K Policy (lru_k.rs) - -## Important Note - -**K is hardcoded to 3 in this implementation** - -```rust -const K: usize = 3; -``` - ---- - -## Core Idea - -Track **last K accesses per frame** and evict based on: - -> Largest backward K-distance - ---- - -## Internal State - -```rust -use std::collections::HashMap; - -pub struct LRUK { - pub history: HashMap>, - pub current_time: u64, -} -``` - ---- - -## record_access() - -```rust -fn record_access(&mut self, frame_id: usize) { - self.current_time += 1; - - let entry = self.history.entry(frame_id).or_insert(Vec::new()); - entry.push(self.current_time); - - if entry.len() > K { - entry.remove(0); // keep only last K - } -} -``` - ---- - -## Explanation - -- Maintain **timestamp history** -- Always keep last **3 accesses** -- Older accesses are removed - ---- - -## victim() - -```rust -fn victim(&mut self, frames: &mut [Frame]) -> Option { - let mut victim = None; - let mut max_distance = 0; - - for (frame_id, times) in &self.history { - if frames[*frame_id].pin_count > 0 { - continue; - } - - let distance = if times.len() < K { - u64::MAX - } else { - self.current_time - times[0] - }; - - if distance > max_distance { - max_distance = distance; - victim = Some(*frame_id); - } - } - - victim -} -``` - ---- - -## Explanation - -### Case 1: Less than K accesses - -```rust -if times.len() < K { - distance = ∞ -} -``` - -Frame is considered **cold** → high eviction priority - ---- - -### Case 2: K accesses available - -```rust -distance = current_time - kth_last_access -``` - Larger distance → less recently used - ---- - -## Example (K = 3) - -```text -Frame histories: - -F1: [5, 10, 20] -F2: [3, 8, 15] -F3: [12, 18] (less than K) - -Current time = 25 -``` - -### Compute distance - -```text -F1 → 25 - 5 = 20 -F2 → 25 - 3 = 22 -F3 → ∞ (highest priority) -``` -Victim = F3 - ---- - -## Diagram - -```text -Time → -|----|----|----|----|----|----|----| - -F1: • • • -F2: • • • -F3: • • - -Evict → frame with oldest 3rd access -``` - ---- - -# 6. Integration with Buffer Pool - -## Flow - -```text -fetch_page(page_id): - - if page exists: - policy.record_access(frame_id) - - else: - victim = policy.victim(frames) - - if victim.is_dirty: - write_page() - - replace victim - load new page -``` - ---- - -# 7. Final Notes - -- All policies: - - Ignore pinned frames - - Work on frame indices -- Buffer manager ensures: - - Dirty pages flushed before eviction - ---- - -# 8. Architectural Context - -These policies belong to the **Buffer Manager Layer**, which minimizes disk I/O -and manages in-memory pages efficiently :contentReference[oaicite:0]{index=0}. - - From d40f4645e242af0071a9d2a10f7488fea629ab13 Mon Sep 17 00:00:00 2001 From: gopendra113 Date: Fri, 24 Apr 2026 02:24:51 +0530 Subject: [PATCH 5/5] updated the sidebar --- .../projects/buffer-manager/api-reference.md | 2 +- .../projects/buffer-manager/architecture.md | 9 ++++++--- .../projects/buffer-manager/buffer-manager.md | 7 ++++--- .../projects/buffer-manager/data-structures.md | 5 +++++ .../projects/buffer-manager/replacement-policies.md | 5 +++++ sidebars.ts | 4 ++++ 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/content/storage-engine/projects/buffer-manager/api-reference.md b/content/storage-engine/projects/buffer-manager/api-reference.md index a90b6d0..228b015 100644 --- a/content/storage-engine/projects/buffer-manager/api-reference.md +++ b/content/storage-engine/projects/buffer-manager/api-reference.md @@ -1,5 +1,5 @@ --- -title: Buffer Manager +title: Api Reference sidebar_position: 3 --- diff --git a/content/storage-engine/projects/buffer-manager/architecture.md b/content/storage-engine/projects/buffer-manager/architecture.md index 2a2849c..ae329ac 100644 --- a/content/storage-engine/projects/buffer-manager/architecture.md +++ b/content/storage-engine/projects/buffer-manager/architecture.md @@ -1,3 +1,8 @@ +--- +title: Architecture +sidebar_position: 1 +--- + # Buffer Manager Architecture (RookDB) This document provides a **detailed architectural overview** of the Buffer Manager in RookDB. @@ -21,8 +26,6 @@ The Buffer Manager is part of the **Storage Manager Layer**, positioned between: ↓ Buffer Manager ↓ - Page Layer - ↓ Disk Storage ``` @@ -421,7 +424,7 @@ General Zone → flexibility The Buffer Manager: -- Sits above the Page Layer +- Sits below the Query Layer / Execution Engine Layer - Manages in-memory caching - Reduces disk access overhead diff --git a/content/storage-engine/projects/buffer-manager/buffer-manager.md b/content/storage-engine/projects/buffer-manager/buffer-manager.md index 977d1e6..b7f01ac 100644 --- a/content/storage-engine/projects/buffer-manager/buffer-manager.md +++ b/content/storage-engine/projects/buffer-manager/buffer-manager.md @@ -97,7 +97,7 @@ This folder contains specialized documentation for different aspects: | **[API Reference](./api-reference.md)** | Complete method signatures and behavior | You need to call a specific method | | **[Data Structures](./data-structures.md)** | Definition of PageId, FrameMetadata, BufferFrame, etc. | Understanding internal types | | **[Architecture](./architecture.md)** | System design, frame layout, reservation strategy | How components fit together | -| **[Replacement Policies](./replacement_policies.md)** | Detailed Clock, LRU, and LRU-K algorithms | Choosing/tuning eviction strategy | +| **[Replacement Policies](./replacement-policies.md)** | Detailed Clock, LRU, and LRU-K algorithms | Choosing/tuning eviction strategy | ### Quick Navigation @@ -105,7 +105,8 @@ This folder contains specialized documentation for different aspects: - **Call a method** → See [API Reference](./api-reference.md) - **Understand a data structure** → See [Data Structures](./data-structures.md) - **Learn how it works internally** → See [Architecture](./architecture.md) -- **Choose a replacement policy** → See [Replacement Policies](./replacement-policies.md) +- **Choose a replacement policy** → See +[Replacement Policies](./replacement-policies.md) - **Get an Overview** → Keep reading this document --- @@ -156,7 +157,7 @@ The buffer manager divides the buffer pool into two regions: ``` 1. INITIALIZATION ├─ Create BufferPool with chosen replacement policy - └─ Register all table files: buffer.register_file("users")? + └─ Register all table files 2. DURING QUERY EXECUTION ├─ fetch_page("users", 0)? ← Get page from buffer diff --git a/content/storage-engine/projects/buffer-manager/data-structures.md b/content/storage-engine/projects/buffer-manager/data-structures.md index e72c7d4..2288754 100644 --- a/content/storage-engine/projects/buffer-manager/data-structures.md +++ b/content/storage-engine/projects/buffer-manager/data-structures.md @@ -1,3 +1,8 @@ +--- +title: Data Structures +sidebar_position: 1 +--- + # Buffer Manager Data Structures This document describes the key data structures used in the Buffer Manager component of RookDB. diff --git a/content/storage-engine/projects/buffer-manager/replacement-policies.md b/content/storage-engine/projects/buffer-manager/replacement-policies.md index c83d81d..b50a12a 100644 --- a/content/storage-engine/projects/buffer-manager/replacement-policies.md +++ b/content/storage-engine/projects/buffer-manager/replacement-policies.md @@ -1,3 +1,8 @@ +--- +title: Replacement Policies +sidebar_position: 1 +--- + # Buffer Manager Replacement Policies ## Overview diff --git a/sidebars.ts b/sidebars.ts index 46dd658..ea1def9 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -49,6 +49,10 @@ const sidebars: SidebarsConfig = { collapsed: true, items: [ "storage-engine/projects/buffer-manager/buffer-manager", + "storage-engine/projects/buffer-manager/replacement-policies", + "storage-engine/projects/buffer-manager/api-reference", + "storage-engine/projects/buffer-manager/architecture", + "storage-engine/projects/buffer-manager/data-structures", ], },