"""Session 42 lexical grouping: join an apostrophe flanked immediately by runes.
The historical word_core.tokenize intentionally remains unchanged for reproduction.
Rune, page and section offsets are preserved; word indices change after a join.
This is a lexical convention, not a claim that apostrophes cannot reset a cipher.
"""
import re
from word_core import R,tokenize
PATTERN=re.compile('(?<=['+R+"])[’'](?=["+R+'])')

def join_intrarune_apostrophes(source):
    return PATTERN.sub('',source)

def tokenize_words(source):
    return tokenize(join_intrarune_apostrophes(source))