# Airflow Migration Sprint Status
# Feature: Migrate from Celery to Apache Airflow
# Tech Spec: docs/feats/airflow-migration-tech-spec.md
# Created: 2025-12-08

project: TextIQ-Doc-Extraction
feature: airflow-migration
status: in_progress
created: 2025-12-08
updated: 2025-12-18
tech_spec: "../airflow-migration-tech-spec.md"

# Epic Overview
epics:
  - id: epic-1
    name: "Airflow Infrastructure & Parallel Deployment"
    status: in_progress
    priority: high
    stories_total: 7
    stories_complete: 6

  - id: epic-2
    name: "Gradual Traffic Migration"
    status: blocked
    blocked_by: epic-1
    priority: high
    stories_total: 7
    stories_complete: 0

  - id: epic-3
    name: "Celery Deprecation & Cleanup"
    status: blocked
    blocked_by: epic-2
    priority: medium
    stories_total: 5
    stories_complete: 0

# Current Sprint
current_sprint:
  name: "Sprint 1 - Infrastructure Setup"
  epic: epic-1
  start_date: 2025-12-08
  end_date: null
  goal: "Deploy Airflow infrastructure and create core operators"

# Story Status
stories:
  # Epic 1: Airflow Infrastructure & Parallel Deployment
  - id: "1.1"
    epic: epic-1
    name: "Airflow Docker Infrastructure Setup"
    status: done
    priority: high
    estimate: "4h"
    completed_date: 2025-12-08
    file: "sprint_artifacts/story-1-1-airflow-infrastructure.md"

  - id: "1.2"
    epic: epic-1
    name: "DAG Project Structure & Configuration"
    status: done
    priority: high
    estimate: "2h"
    depends_on: ["1.1"]
    completed_date: 2025-12-09
    file: "sprint_artifacts/story-1-2-dag-project-structure.md"

  - id: "1.3"
    epic: epic-1
    name: "Core Operators (Validate, Fetch, Parse)"
    status: done
    priority: high
    estimate: "8h"
    depends_on: ["1.2"]
    completed_date: 2025-12-09
    file: "sprint_artifacts/story-1-3-core-operators.md"

  - id: "1.4"
    epic: epic-1
    name: "Chunking & Embedding Tasks"
    status: done
    priority: high
    estimate: "6h"
    depends_on: ["1.3"]
    completed_date: 2025-12-09
    file: "sprint_artifacts/story-1-4-chunking-embedding-operators.md"
    notes: "Refactored to use @task.external_python instead of operators"

  - id: "1.5"
    epic: epic-1
    name: "Storage Tasks (Milvus only)"
    status: done
    priority: high
    estimate: "4h"
    depends_on: ["1.4"]
    completed_date: 2025-12-09
    file: "sprint_artifacts/story-1-5-storage-operators.md"
    notes: "Milvus only - PostgreSQL store_chunks_task removed; all tasks use pickle files for large data transfer"

  - id: "1.6"
    epic: epic-1
    name: "Main DAG Definition & Branching Logic"
    status: done
    priority: high
    estimate: "4h"
    depends_on: ["1.5"]
    completed_date: 2025-12-09
    file: "sprint_artifacts/story-1-6-main-dag-definition.md"
    notes: "Using TaskFlow API with @task decorators"

  - id: "1.7"
    epic: epic-1
    name: "Shadow Mode Implementation & Metrics"
    status: pending
    priority: medium
    estimate: "6h"
    depends_on: ["1.6"]
    file: "sprint_artifacts/story-1-7-shadow-mode.md"

  # Epic 2: Gradual Traffic Migration
  - id: "2.1"
    epic: epic-2
    name: "Feature Flag Infrastructure"
    status: pending
    priority: high
    estimate: "3h"
    depends_on: ["1.7"]
    file: "sprint_artifacts/epic-2-traffic-migration.md#story-21"

  - id: "2.2"
    epic: epic-2
    name: "Airflow REST API Client Integration"
    status: pending
    priority: high
    estimate: "4h"
    depends_on: ["2.1"]
    file: "sprint_artifacts/epic-2-traffic-migration.md#story-22"

  - id: "2.3"
    epic: epic-2
    name: "API Route Modification for Traffic Splitting"
    status: pending
    priority: high
    estimate: "4h"
    depends_on: ["2.2"]
    file: "sprint_artifacts/epic-2-traffic-migration.md#story-23"

  - id: "2.4"
    epic: epic-2
    name: "Monitoring Dashboard & Alerts"
    status: pending
    priority: medium
    estimate: "4h"
    depends_on: ["2.1"]
    file: "sprint_artifacts/epic-2-traffic-migration.md#story-24"

  - id: "2.5"
    epic: epic-2
    name: "Canary Deployment (10% Traffic)"
    status: pending
    priority: high
    estimate: "2h"
    depends_on: ["2.3", "2.4"]
    file: "sprint_artifacts/epic-2-traffic-migration.md#story-25"

  - id: "2.6"
    epic: epic-2
    name: "Expanded Rollout (30% → 50% → 80%)"
    status: pending
    priority: high
    estimate: "3h"
    depends_on: ["2.5"]
    file: "sprint_artifacts/epic-2-traffic-migration.md#story-26"

  - id: "2.7"
    epic: epic-2
    name: "Full Migration (100% Traffic)"
    status: pending
    priority: high
    estimate: "2h"
    depends_on: ["2.6"]
    file: "sprint_artifacts/epic-2-traffic-migration.md#story-27"

  # Epic 3: Celery Deprecation & Cleanup
  - id: "3.1"
    epic: epic-3
    name: "Verify 100% Airflow Stability (1 Week Soak)"
    status: pending
    priority: high
    estimate: "ongoing"
    depends_on: ["2.7"]
    file: "sprint_artifacts/epic-3-celery-deprecation.md#story-31"

  - id: "3.2"
    epic: epic-3
    name: "Remove Celery Extraction Task"
    status: pending
    priority: medium
    estimate: "2h"
    depends_on: ["3.1"]
    file: "sprint_artifacts/epic-3-celery-deprecation.md#story-32"

  - id: "3.3"
    epic: epic-3
    name: "Archive Legacy Code"
    status: pending
    priority: low
    estimate: "1h"
    depends_on: ["3.2"]
    file: "sprint_artifacts/epic-3-celery-deprecation.md#story-33"

  - id: "3.4"
    epic: epic-3
    name: "Update API & Documentation"
    status: pending
    priority: medium
    estimate: "3h"
    depends_on: ["3.2"]
    file: "sprint_artifacts/epic-3-celery-deprecation.md#story-34"

  - id: "3.5"
    epic: epic-3
    name: "Final Cleanup & Validation"
    status: pending
    priority: medium
    estimate: "2h"
    depends_on: ["3.3", "3.4"]
    file: "sprint_artifacts/epic-3-celery-deprecation.md#story-35"

# Estimate Summary
estimates:
  epic_1_total: "34h"
  epic_2_total: "22h"
  epic_3_total: "8h + 1 week soak"
  total: "64h + 1 week soak"

# Success Metrics (from tech spec)
success_metrics:
  processing_success_rate:
    current: "95%"
    target: "≥95%"
  avg_processing_time:
    current: "45s"
    target: "≤50s"
  p99_processing_time:
    current: "180s"
    target: "≤200s"
  developer_debugging_time:
    current: "30min avg"
    target: "≤15min"

# Notes
notes:
  - "Stories 1.1-1.7 can be done in parallel with minimal dependencies"
  - "Epic 2 requires Epic 1 complete with shadow mode validated"
  - "Epic 3 requires 1 week soak period at 100% traffic"
  - "Rollback plan documented in tech spec"
