bible-epub-processing（圣经 epub 处理）

Bible EPUB Processing

Overview

Parse and extract structured content from New World Translation (NWT) Bible EPUBs to build parallel corpora for Chuukese-English translation training. The NWT is available in both Chuukese (nwt_TE.epub) and English (nwt_E.epub), providing high-quality aligned translations.

File Locations

data/bible/
├── nwt_E.epub      # English NWT Bible
└── nwt_TE.epub     # Chuukese (Trukese) NWT Bible

EPUB Structure

NWT EPUBs follow a specific structure:

nwt_X.epub/
├── META-INF/
│   └── container.xml
├── OEBPS/
│   ├── content.opf
│   ├── toc.ncx
│   └── OEBPS/
│       ├── 01_Genesis.xhtml
│       ├── 02_Exodus.xhtml
│       └── ... (66 books)

NWT EPUB Parser

Core Parser Class

# src/utils/nwt_epub_parser.py
import ebooklib
from ebooklib import epub
from bs4 import BeautifulSoup
import re
from typing import Dict, List, Optional, Tuple
import logging

logger = logging.getLogger(__name__)

class NWTEpubParser:
    """
    Parser for New World Translation Bible EPUBs.
    Extracts books, chapters, and verses with proper structure.
    """

    # Book order in NWT (66 books)
    BOOK_ORDER = [
        "Genesis", "Exodus", "Leviticus", "Numbers", "Deuteronomy",
        "Joshua", "Judges", "Ruth", "1 Samuel", "2 Samuel",
        "1 Kings", "2 Kings", "1 Chronicles", "2 Chronicles",
        "Ezra", "Nehemiah", "Esther", "Job", "Psalms", "Proverbs",
        "Ecclesiastes", "Song of Solomon", "Isaiah", "Jeremiah",
        "Lamentations", "Ezekiel", "Daniel", "Hosea", "Joel", "Amos",
        "Obadiah", "Jonah", "Micah", "Nahum", "Habakkuk", "Zephaniah",
        "Haggai", "Zechariah", "Malachi", "Matthew", "Mark", "Luke",
        "John", "Acts", "Romans", "1 Corinthians", "2 Corinthians",
        "Galatians", "Ephesians", "Philippians", "Colossians",
        "1 Thessalonians", "2 Thessalonians", "1 Timothy", "2 Timothy",
        "Titus", "Philemon", "Hebrews", "James", "1 Peter", "2 Peter",
        "1 John", "2 John", "3 John", "Jude", "Revelation"
    ]

    def __init__(self, epub_path: str):
        self.epub_path = epub_path
        self.book = epub.read_epub(epub_path)
        self._book_cache = {}
        self._chapter_cache = {}

        logger.info(f"Loaded EPUB: {epub_path}")

    def get_book_list(self) -> List[str]:
        """Get list of available books in the EPUB."""
        books = []

        for item in self.book.get_items():
            if item.get_type() == ebooklib.ITEM_DOCUMENT:
                name = item.get_name()
                # Extract book name from filename
                match = re.search(r'\d+_(\w+)\.xhtml', name)
                if match:
                    books.append(match.group(1))

        return books

    def get_chapters(self, book_name: str) -> List[int]:
        """Get list of chapter numbers for a book."""
        content = self._get_book_content(book_name)
        if not content:
            return []

        soup = BeautifulSoup(content, 'lxml')

        # Find chapter markers
        chapters = set()
        for elem in soup.find_all(class_=re.compile(r'chapter|chapterNum')):
            text = elem.get_text(strip=True)
            try:
                chapter_num = int(re.search(r'\d+', text).group())
                chapters.add(chapter_num)
            except (ValueError, AttributeError):
                continue

        return sorted(chapters)

    def get_verses(self, book_name: str, chapter: int) -> Dict[int, str]:
        """
        Get all verses for a specific chapter.

        Returns:
            Dictionary mapping verse numbers to verse text
        """
        content = self._get_book_content(book_name)
        if not content:
            return {}

        soup = BeautifulSoup(content, 'lxml')
        verses = {}

        # Find the chapter section
        chapter_section = self._find_chapter_section(soup, chapter)
        if not chapter_section:
            return {}

        # Extract verses
        for verse_elem in chapter_section.find_all(class_=re.compile(r'verse|v\d+')):
            verse_num = self._extract_verse_number(verse_elem)
            if verse_num:
                verse_text = self._extract_verse_text(verse_elem)
                if verse_text:
                    verses[verse_num] = verse_text

        return verses

    def get_verse(self, book_name: str, chapter: int, verse: int) -> Optional[str]:
        """Get a specific verse."""
        verses = self.get_verses(book_name, chapter)
        return verses.get(verse)

    def get_verse_range(
        self,
        book_name: str,
        chapter: int,
        start_verse: int,
        end_verse: int
    ) -> str:
        """Get a range of verses as combined text."""
        verses = self.get_verses(book_name, chapter)

        verse_texts = []
        for v in range(start_verse, end_verse + 1):
            if v in verses:
                verse_texts.append(verses[v])

        return ' '.join(verse_texts)

    def _get_book_content(self, book_name: str) -> Optional[str]:
        """Get raw HTML content for a book."""
        if book_name in self._book_cache:
            return self._book_cache[book_name]

        for item in self.book.get_items():
            if item.get_type() == ebooklib.ITEM_DOCUMENT:
                if book_name.lower() in item.get_name().lower():
                    content = item.get_content().decode('utf-8')
                    self._book_cache[book_name] = content
                    return content

        return None

    def _find_chapter_section(self, soup: BeautifulSoup, chapter: int):
        """Find the DOM section for a specific chapter."""
        # Different strategies for finding chapter boundaries

        # Strategy 1: Look for chapter heading
        for heading in soup.find_all(['h2', 'h3', 'div'], class_=re.compile(r'chapter')):
            if str(chapter) in heading.get_text():
                return heading.find_parent('section') or heading.find_parent('div')

        # Strategy 2: Look for chapter number span
        for span in soup.find_all('span', class_=re.compile(r'chapterNum')):
            if span.get_text(strip=True) == str(chapter):
                return span.find_parent('section') or span.find_parent('div')

        return soup  # Return whole document if chapter not found

    def _extract_verse_number(self, elem) -> Optional[int]:
        """Extract verse number from element."""
        # Check for verse number class
        verse_class = elem.get('class', [])
        for cls in verse_class:
            match = re.search(r'v(\d+)', cls)
            if match:
                return int(match.group(1))

        # Check for data attribute
        verse_num = elem.get('data-verse')
        if verse_num:
            return int(verse_num)

        # Check for sup element with verse number
        sup = elem.find('sup')
        if sup:
            try:
                return int(sup.get_text(strip=True))
            except ValueError:
                pass

        return None

    def _extract_verse_text(self, elem) -> str:
        """Extract clean verse text from element."""
        # Remove verse number and footnote markers
        text = elem.get_text(separator=' ', strip=True)

        # Clean up the text
        text = re.sub(r'^\d+\s*', '', text)  # Remove leading verse number
        text = re.sub(r'\*+', '', text)  # Remove footnote markers
        text = re.sub(r'\s+', ' ', text)  # Normalize whitespace

        return text.strip()

Parallel Corpus Builder

class ParallelCorpusBuilder:
    """
    Build parallel corpora from Chuukese and English NWT EPUBs.
    """

    def __init__(self, chuukese_epub: str, english_epub: str):
        self.chk_parser = NWTEpubParser(chuukese_epub)
        self.en_parser = NWTEpubParser(english_epub)

    def build_parallel_corpus(
        self,
        books: List[str] = None,
        min_length: int = 10,
        max_length: int = 500
    ) -> List[Dict]:
        """
        Build parallel verse pairs.

        Args:
            books: List of books to include (None = all)
            min_length: Minimum verse length in characters
            max_length: Maximum verse length in characters

        Returns:
            List of parallel pairs with metadata
        """
        if books is None:
            books = self.chk_parser.get_book_list()

        pairs = []

        for book in books:
            logger.info(f"Processing {book}...")

            chapters = self.chk_parser.get_chapters(book)

            for chapter in chapters:
                chk_verses = self.chk_parser.get_verses(book, chapter)
                en_verses = self.en_parser.get_verses(book, chapter)

                # Align verses by number
                for verse_num in chk_verses:
                    if verse_num in en_verses:
                        chk_text = chk_verses[verse_num]
                        en_text = en_verses[verse_num]

                        # Filter by length
                        if (min_length <= len(chk_text) <= max_length and
                            min_length <= len(en_text) <= max_length):

                            pairs.append({
                                'chuukese': chk_text,
                                'english': en_text,
                                'reference': f"{book} {chapter}:{verse_num}",
                                'book': book,
                                'chapter': chapter,
                                'verse': verse_num,
                                'source': 'nwt_bible',
                                'confidence': 0.95  # High quality translations
                            })

        logger.info(f"Built {len(pairs)} parallel pairs")
        return pairs

    def export_for_training(
        self,
        output_dir: str,
        direction: str = "chk_to_en",
        test_split: float = 0.1,
        val_split: float = 0.1
    ) -> Dict[str, int]:
        """
        Export parallel corpus for model training.

        Args:
            output_dir: Directory for output files
            direction: 'chk_to_en' or 'en_to_chk'
            test_split: Fraction for test set
            val_split: Fraction for validation set

        Returns:
            Statistics about exported data
        """
        import os
        from sklearn.model_selection import train_test_split

        os.makedirs(output_dir, exist_ok=True)

        pairs = self.build_parallel_corpus()

        # Split data
        train, test = train_test_split(pairs, test_size=test_split, random_state=42)
        train, val = train_test_split(train, test_size=val_split/(1-test_split), random_state=42)

        # Export TSV files
        for split_name, split_data in [('train', train), ('val', val), ('test', test)]:
            filepath = os.path.join(output_dir, f'{split_name}.tsv')

            with open(filepath, 'w', encoding='utf-8') as f:
                for pair in split_data:
                    if direction == "chk_to_en":
                        source = pair['chuukese']
                        target = pair['english']
                    else:
                        source = pair['english']
                        target = pair['chuukese']

                    f.write(f"{source}\t{target}\n")

        return {
            'total': len(pairs),
            'train': len(train),
            'val': len(val),
            'test': len(test)
        }

    def export_jsonl(self, output_path: str, format_type: str = "ollama") -> int:
        """
        Export parallel corpus as JSONL for LLM training.

        Args:
            output_path: Path for output file
            format_type: 'ollama', 'openai', or 'huggingface'

        Returns:
            Number of examples exported
        """
        import json

        pairs = self.build_parallel_corpus()

        with open(output_path, 'w', encoding='utf-8') as f:
            for pair in pairs:
                if format_type == "ollama":
                    example = {
                        "prompt": f"Translate this Chuukese verse to English: {pair['chuukese']}",
                        "response": pair['english'],
                        "system": "You are a Chuukese-English Bible translator."
                    }
                elif format_type == "openai":
                    example = {
                        "messages": [
                            {"role": "system", "content": "You are a Chuukese-English translator."},
                            {"role": "user", "content": f"Translate: {pair['chuukese']}"},
                            {"role": "assistant", "content": pair['english']}
                        ]
                    }
                else:  # huggingface
                    example = {
                        "text": f"### Instruction:\nTranslate this Chuukese text to English.\n\n### Input:\n{pair['chuukese']}\n\n### Response:\n{pair['english']}"
                    }

                f.write(json.dumps(example, ensure_ascii=False) + '\n')

        return len(pairs)

Usage Examples

Basic Verse Extraction

from src.utils.nwt_epub_parser import NWTEpubParser

# Load English Bible
parser = NWTEpubParser('data/bible/nwt_E.epub')

# Get list of books
books = parser.get_book_list()
print(f"Found {len(books)} books")

# Get chapters in Genesis
chapters = parser.get_chapters('Genesis')
print(f"Genesis has {len(chapters)} chapters")

# Get all verses in Genesis 1
verses = parser.get_verses('Genesis', 1)
print(f"Genesis 1 has {len(verses)} verses")

# Get specific verse
verse = parser.get_verse('Genesis', 1, 1)
print(f"Genesis 1:1 - {verse}")

Building Parallel Corpus

from src.utils.nwt_epub_parser import ParallelCorpusBuilder

# Build corpus from both Bibles
builder = ParallelCorpusBuilder(
    chuukese_epub='data/bible/nwt_TE.epub',
    english_epub='data/bible/nwt_E.epub'
)

# Get all parallel pairs
pairs = builder.build_parallel_corpus()
print(f"Total parallel pairs: {len(pairs)}")

# Export for Helsinki-NLP training
stats = builder.export_for_training(
    output_dir='training_data/bible',
    direction='chk_to_en'
)
print(f"Train: {stats['train']}, Val: {stats['val']}, Test: {stats['test']}")

Exporting for LLM Training

# Export for Ollama fine-tuning
count = builder.export_jsonl(
    output_path='training_data/bible_ollama.jsonl',
    format_type='ollama'
)
print(f"Exported {count} examples for Ollama")

Quality Considerations

High-Quality Pairs

• Bible translations are professionally reviewed
• Consistent terminology across verses
• Good for formal/religious register

Potential Issues

• Religious vocabulary may not generalize
• Formal language style
• Some verses have complex structures

Filtering Strategies

def filter_pairs(pairs):
    """Filter pairs for training quality."""
    filtered = []

    for pair in pairs:
        # Skip very short pairs
        if len(pair['chuukese']) < 20 or len(pair['english']) < 20:
            continue

        # Skip pairs with unusual length ratios
        ratio = len(pair['chuukese']) / len(pair['english'])
        if ratio < 0.5 or ratio > 2.0:
            continue

        # Skip pairs with mostly numbers
        if sum(c.isdigit() for c in pair['chuukese']) / len(pair['chuukese']) > 0.3:
            continue

        filtered.append(pair)

    return filtered

Dependencies

• ebooklib>=0.18: EPUB parsing
• beautifulsoup4>=4.12.0: HTML parsing
• lxml>=4.9.0: XML/HTML parser
• scikit-learn>=1.0.0: Data splitting (optional)