import re from collections import Counter, defaultdict import random def tokenize(text): return re.findall(r"\w+|[^\w\s]", text.lower()) def is_punctuation(tok): # Punctuation-only tokens (e.g., ",", ".", "!", "?", ":") return re.match(r"^[^\w\s]+$", tok) is not None def build_model(corpus_tokens, window=3): """Fixed-window next-token model with explicit ~~and~~ .""" model = defaultdict(Counter) start_pad = ["~~"] * window end_pad = ["~~"] * window seq = start_pad + corpus_tokens + end_pad for i in range(len(seq) - window): context = tuple(seq[i:i + window]) next_tok = seq[i + window] model[context][next_tok] += 1 return model def sample_next(counter, rng): tokens = list(counter.keys()) counts = [counter[t] for t in tokens] total = sum(counts) r = rng.random() * total cum = 0 for tok, c in zip(tokens, counts): cum += c if r <= cum: return tok return tokens[-1] def generate(prompt, model, window=3, max_new_tokens=30, seed=0): rng = random.Random(seed) tokens = tokenize(prompt) if len(tokens) < window: tokens = [""] * (window - len(tokens)) + tokens for _ in range(max_new_tokens): context = tuple(tokens[-window:]) next_counts = model.get(context) if not next_counts: break # Try sampling with constraint: no two consecutive punctuation tokens. prev_tok = tokens[-1] if tokens else None prev_is_punct = is_punctuation(prev_tok) if prev_tok is not None else False chosen = None for _attempt in range(20): cand = sample_next(next_counts, rng) if prev_is_punct and is_punctuation(cand): continue chosen = cand break if chosen is None: # Fallback: most frequent valid token. valid = [] for tok, cnt in next_counts.items(): if prev_is_punct and is_punctuation(tok): continue valid.append((cnt, tok)) if not valid: # If everything is invalid, allow the most frequent token. chosen = max(next_counts.items(), key=lambda kv: kv[1])[0] else: chosen = max(valid)[1] tokens.append(chosen) if chosen == "": break out = " ".join(tokens) out = out.replace(" ~~", " ").replace("~~ ", " ") out = out.strip() return out if __name__ == "__main__": corpus = ( "Natural language processing allows programs to read and write human languages. " "Transformers use attention mechanisms for modern NLP. " "Attention helps models focus on relevant tokens during generation." ) corpus_tokens = tokenize(corpus) model = build_model(corpus_tokens, window=3) prompt = "transformers use attention" completion = generate(prompt, model, window=3, max_new_tokens=25, seed=7) print("Prompt:", prompt) print("Completion:", completion)

Artificial Intelligence (AI): goals, methods, applications, and foundations

Content

AI Audio

Test Yourself

Practice

Games

AI Podcast

Educational Podcast