NLTK

graph TD
    A[NLTK] --> B[Corpora]
    A --> C[Tokenization]
    A --> D[Tagging]
    A --> E[Parsing]
    A --> F[Classification]
    A --> G[Semantic Analysis]
    A --> H[Stemming & Lemmatization]
    A --> I[Utilities]

    B --> B1[Text Corpora]
    B --> B2[Lexical Resources]
    B --> B3[WordNet]
    B --> B4[Treebanks]

    C --> C1[Word Tokenization]
    C --> C2[Sentence Tokenization]
    C --> C3[Regex Tokenization]
    C --> C4[Treebank Tokenization]

    D --> D1[POS Tagging]
    D --> D2[Named Entity Recognition]
    D --> D3[Chunking]
    D --> D4[IOB Tagging]

    E --> E1[Context-Free Grammar]
    E --> E2[Dependency Parsing]
    E --> E3[Chart Parsing]
    E --> E4[Probabilistic Parsing]

    F --> F1[Text Classification]
    F --> F2[Sentiment Analysis]
    F --> F3[Document Classification]
    F --> F4[Feature Extraction]

    G --> G1[Word Sense Disambiguation]
    G --> G2[Semantic Similarity]
    G --> G3[Logic & Inference]
    G --> G4[Discourse Analysis]

    H --> H1[Porter Stemmer]
    H --> H2[Lancaster Stemmer]
    H --> H3[Snowball Stemmer]
    H --> H4[WordNet Lemmatizer]

    I --> I1[Frequency Distributions]
    I --> I2[Concordance]
    I --> I3[Collocations]
    I --> I4[Text Utilities]

    style A fill:#8E44AD,stroke:#333
    style B fill:#3498DB,stroke:#333
    style C fill:#2ECC71,stroke:#333
    style D fill:#E74C3C,stroke:#333
    style E fill:#F39C12,stroke:#333
    style F fill:#1ABC9C,stroke:#333
    style G fill:#9B59B6,stroke:#333
    style H fill:#E67E22,stroke:#333
    style I fill:#34495E,stroke:#333

# Basic NLTK example
import nltk
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.probability import FreqDist
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer, WordNetLemmatizer
import matplotlib.pyplot as plt

print("Basic NLTK Example...")

# Download required NLTK data (uncomment if running for the first time)
# nltk.download('punkt')
# nltk.download('stopwords')
# nltk.download('wordnet')
# nltk.download('averaged_perceptron_tagger')

# 1. Text tokenization
print("\n1. Text Tokenization...")
text = "Natural Language Processing (NLP) is a subfield of artificial intelligence. It focuses on the interaction between computers and humans through natural language."

# Sentence tokenization
sentences = sent_tokenize(text)
print(f"Sentences: {sentences}")

# Word tokenization
words = word_tokenize(text)
print(f"Words: {words}")

# 2. Frequency distribution
print("\n2. Frequency Distribution...")
fdist = FreqDist(words)
print(f"Most common words: {fdist.most_common(10)}")

# Plot frequency distribution
plt.figure(figsize=(12, 6))
fdist.plot(30, cumulative=False)
plt.title('Word Frequency Distribution')
plt.show()

# 3. Stopword removal
print("\n3. Stopword Removal...")
stop_words = set(stopwords.words('english'))
filtered_words = [word for word in words if word.lower() not in stop_words]
print(f"Filtered words: {filtered_words}")

# Frequency distribution without stopwords
fdist_filtered = FreqDist(filtered_words)
print(f"Most common words (no stopwords): {fdist_filtered.most_common(10)}")

plt.figure(figsize=(12, 6))
fdist_filtered.plot(30, cumulative=False)
plt.title('Word Frequency Distribution (No Stopwords)')
plt.show()

# 4. Stemming
print("\n4. Stemming...")
stemmer = PorterStemmer()
stemmed_words = [stemmer.stem(word) for word in filtered_words]
print(f"Stemmed words: {stemmed_words}")

# 5. Lemmatization
print("\n5. Lemmatization...")
lemmatizer = WordNetLemmatizer()
lemmatized_words = [lemmatizer.lemmatize(word) for word in filtered_words]
print(f"Lemmatized words: {lemmatized_words}")

# Compare stemming and lemmatization
print("\nComparison of Stemming and Lemmatization:")
for original, stemmed, lemmatized in zip(filtered_words[:10], stemmed_words[:10], lemmatized_words[:10]):
    print(f"{original:15} -> {stemmed:15} -> {lemmatized:15}")

# 6. Part-of-Speech tagging
print("\n6. Part-of-Speech Tagging...")
pos_tags = nltk.pos_tag(words)
print(f"POS tags: {pos_tags}")

# 7. Named Entity Recognition
print("\n7. Named Entity Recognition...")
# Download required data if not already present
try:
    nltk.data.find('chunkers/maxent_ne_chunker')
except LookupError:
    nltk.download('maxent_ne_chunker')
    nltk.download('words')

# Sample text for NER
ner_text = "Apple Inc. is planning to open a new store in Paris next month. Tim Cook will be attending the event."
ner_words = word_tokenize(ner_text)
ner_pos_tags = nltk.pos_tag(ner_words)
ner_chunks = nltk.ne_chunk(ner_pos_tags)

print("Named Entities:")
for chunk in ner_chunks:
    if hasattr(chunk, 'label'):
        print(f"{' '.join(c[0] for c in chunk):<30} {chunk.label()}")

# 8. Concordance
print("\n8. Concordance...")
from nltk.text import Text

# Create a Text object
text_obj = Text(words)

# Find concordance for "language"
print("Concordance for 'language':")
text_obj.concordance("language", width=80, lines=5)

# 9. Collocations
print("\n9. Collocations...")
print("Common bigrams:")
text_obj.collocations(num=10, window_size=2)

# 10. Similar words
print("\n10. Similar Words...")
print("Words similar to 'language':")
text_obj.similar("language")

# Text classification example with NLTK
import nltk
import random
from nltk.corpus import movie_reviews
from nltk.classify import NaiveBayesClassifier, accuracy
from nltk.classify.util import apply_features
from nltk.tokenize import word_tokenize
from nltk.probability import FreqDist
import matplotlib.pyplot as plt

print("\nText Classification Example...")

# 1. Load dataset
print("Loading dataset...")
try:
    nltk.data.find('corpora/movie_reviews')
except LookupError:
    nltk.download('movie_reviews')

documents = [(list(movie_reviews.words(fileid)), category)
             for category in movie_reviews.categories()
             for fileid in movie_reviews.fileids(category)]

random.shuffle(documents)
print(f"Loaded {len(documents)} documents")

# 2. Feature extraction
print("\nFeature extraction...")
all_words = FreqDist(w.lower() for w in movie_reviews.words())
word_features = list(all_words.keys())[:2000]

def document_features(document):
    document_words = set(document)
    features = {}
    for word in word_features:
        features[f'contains({word})'] = (word in document_words)
    return features

# Example feature extraction
print("Example features for first document:")
first_features = document_features(documents[0][0])
print({k: v for k, v in list(first_features.items())[:10]})

# 3. Prepare training and testing sets
print("\nPreparing training and testing sets...")
featuresets = apply_features(document_features, documents, labeled=True)
train_set, test_set = featuresets[:1600], featuresets[1600:]

print(f"Training set size: {len(train_set)}")
print(f"Test set size: {len(test_set)}")

# 4. Train Naive Bayes classifier
print("\nTraining Naive Bayes classifier...")
classifier = NaiveBayesClassifier.train(train_set)

# 5. Evaluate classifier
print("\nEvaluating classifier...")
accuracy_score = accuracy(classifier, test_set)
print(f"Accuracy: {accuracy_score:.4f}")

# Show most informative features
print("\nMost informative features:")
classifier.show_most_informative_features(20)

# 6. Classify new text
print("\nClassifying new text...")
new_texts = [
    "This movie was absolutely fantastic! The acting was superb and the plot was engaging.",
    "I hated every minute of this film. The dialogue was terrible and the acting was wooden.",
    "It was okay, not great but not terrible either. Just an average movie experience.",
    "The cinematography was beautiful and the story was compelling.",
    "Waste of time and money. I want my money back!"
]

for text in new_texts:
    features = document_features(word_tokenize(text))
    prediction = classifier.classify(features)
    print(f"Text: {text}")
    print(f"Predicted sentiment: {prediction}")
    print(f"Probability: {classifier.prob_classify(features).prob(prediction):.4f}")
    print()

# 7. Error analysis
print("\nError analysis...")
errors = []
for (text, tag) in test_set:
    guess = classifier.classify(text)
    if guess != tag:
        errors.append((tag, guess, text))

print(f"Found {len(errors)} errors")
print("Sample errors:")
for (tag, guess, text) in errors[:5]:
    print(f"Actual: {tag}, Predicted: {guess}")
    print(f"Features: {list(text.keys())[:10]}")
    print()

# 8. Feature importance visualization
print("\nFeature importance visualization...")
# Get feature counts for each class
pos_features = classifier._feature_probdist[1]  # Positive class
neg_features = classifier._feature_probdist[0]  # Negative class

# Get top features for each class
top_pos_features = sorted(pos_features.freqdist().items(), key=lambda x: x[1], reverse=True)[:20]
top_neg_features = sorted(neg_features.freqdist().items(), key=lambda x: x[1], reverse=True)[:20]

# Prepare data for visualization
pos_words = [word.replace('contains(', '').replace(')', '') for word, _ in top_pos_features]
pos_counts = [count for _, count in top_pos_features]

neg_words = [word.replace('contains(', '').replace(')', '') for word, _ in top_neg_features]
neg_counts = [count for _, count in top_neg_features]

# Plot
plt.figure(figsize=(15, 8))

plt.subplot(1, 2, 1)
plt.barh(pos_words, pos_counts, color='green')
plt.title('Top Positive Features')
plt.xlabel('Frequency')

plt.subplot(1, 2, 2)
plt.barh(neg_words, neg_counts, color='red')
plt.title('Top Negative Features')
plt.xlabel('Frequency')

plt.tight_layout()
plt.show()

# Corpus analysis example with NLTK
import nltk
from nltk.corpus import brown, reuters, inaugural
from nltk.probability import FreqDist, ConditionalFreqDist
from nltk.tokenize import word_tokenize
import matplotlib.pyplot as plt

print("\nCorpus Analysis Example...")

# 1. Load different corpora
print("Loading corpora...")
try:
    nltk.data.find('corpora/brown')
    nltk.data.find('corpora/reuters')
    nltk.data.find('corpora/inaugural')
except LookupError:
    nltk.download('brown')
    nltk.download('reuters')
    nltk.download('inaugural')

# Brown Corpus (various genres)
print(f"Brown corpus categories: {brown.categories()}")
print(f"Brown corpus file count: {len(brown.fileids())}")

# Reuters Corpus (news articles)
print(f"Reuters corpus categories: {reuters.categories()[:10]}...")
print(f"Reuters corpus file count: {len(reuters.fileids())}")

# Inaugural Corpus (presidential addresses)
print(f"Inaugural corpus file count: {len(inaugural.fileids())}")
print(f"Inaugural corpus years: {[fileid[:4] for fileid in inaugural.fileids()]}")

# 2. Genre analysis with Brown Corpus
print("\n2. Genre Analysis with Brown Corpus...")
genres = ['news', 'religion', 'hobbies', 'science_fiction', 'romance']
genre_words = []

for genre in genres:
    words = brown.words(categories=genre)
    genre_words.append((genre, words[:2000]))  # Limit to first 2000 words for demo

# Create frequency distributions
cfd = ConditionalFreqDist(
    (genre, word.lower())
    for genre, words in genre_words
    for word in words
    if word.isalpha()  # Only alphabetic words
)

# Plot most common words by genre
plt.figure(figsize=(15, 10))
cfd.plot(20, title='Most Common Words by Genre')
plt.show()

# 3. Temporal analysis with Inaugural Corpus
print("\n3. Temporal Analysis with Inaugural Corpus...")
cfd_time = ConditionalFreqDist(
    (fileid[:4], word.lower())  # Use year as condition
    for fileid in inaugural.fileids()
    for word in inaugural.words(fileid)
    if word.isalpha()
)

# Plot word usage over time
plt.figure(figsize=(15, 8))
cfd_time.plot(30, title='Word Usage Over Time in Inaugural Addresses')
plt.show()

# Track specific words over time
words_to_track = ['freedom', 'war', 'peace', 'america', 'nation']
plt.figure(figsize=(15, 8))

for word in words_to_track:
    years = [year for year in cfd_time.conditions() if word in cfd_time[year]]
    counts = [cfd_time[year][word] for year in years]
    plt.plot(years, counts, label=word, marker='o')

plt.title('Word Usage Trends in Inaugural Addresses')
plt.xlabel('Year')
plt.ylabel('Frequency')
plt.legend()
plt.grid(True)
plt.show()

# 4. Topic analysis with Reuters Corpus
print("\n4. Topic Analysis with Reuters Corpus...")
topics = ['earn', 'acq', 'money-fx', 'grain', 'crude']
topic_words = []

for topic in topics:
    fileids = reuters.fileids(topic)
    words = reuters.words(fileids[:5])  # Use first 5 files for demo
    topic_words.append((topic, words[:2000]))  # Limit to first 2000 words

# Create frequency distributions
cfd_topic = ConditionalFreqDist(
    (topic, word.lower())
    for topic, words in topic_words
    for word in words
    if word.isalpha()
)

# Plot most common words by topic
plt.figure(figsize=(15, 10))
cfd_topic.plot(20, title='Most Common Words by Topic in Reuters Corpus')
plt.show()

# 5. Comparative analysis
print("\n5. Comparative Analysis...")
# Compare word usage between genres
print("Comparing word usage between genres:")
for word in ['god', 'science', 'love', 'money', 'war']:
    print(f"\nWord: {word}")
    for genre in genres:
        count = cfd[genre][word]
        print(f"{genre:15}: {count}")

# 6. Lexical diversity analysis
print("\n6. Lexical Diversity Analysis...")
def lexical_diversity(words):
    return len(set(words)) / len(words)

print("Lexical diversity by genre:")
for genre, words in genre_words:
    diversity = lexical_diversity([w.lower() for w in words if w.isalpha()])
    print(f"{genre:15}: {diversity:.4f}")

print("\nLexical diversity by topic:")
for topic, words in topic_words:
    diversity = lexical_diversity([w.lower() for w in words if w.isalpha()])
    print(f"{topic:15}: {diversity:.4f}")

# 7. Collocation analysis
print("\n7. Collocation Analysis...")
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import BigramAssocMeasures

# Analyze collocations in news genre
news_words = brown.words(categories='news')
news_words = [word.lower() for word in news_words if word.isalpha()]

finder = BigramCollocationFinder.from_words(news_words)
finder.apply_freq_filter(5)  # Only consider bigrams that occur at least 5 times

print("Top collocations in news genre:")
print(finder.nbest(BigramAssocMeasures.likelihood_ratio, 20))

# Analyze collocations in romance genre
romance_words = brown.words(categories='romance')
romance_words = [word.lower() for word in romance_words if word.isalpha()]

finder = BigramCollocationFinder.from_words(romance_words)
finder.apply_freq_filter(5)

print("\nTop collocations in romance genre:")
print(finder.nbest(BigramAssocMeasures.likelihood_ratio, 20))

# Syntax parsing example with NLTK
import nltk
from nltk import CFG, ChartParser, RecursiveDescentParser
from nltk.tokenize import word_tokenize
from nltk.draw import TreeView
import matplotlib.pyplot as plt

print("\nSyntax Parsing Example...")

# 1. Context-Free Grammar
print("1. Context-Free Grammar...")
grammar = CFG.fromstring("""
    S -> NP VP
    NP -> Det N | 'I'
    VP -> V NP | V NP PP
    PP -> P NP
    Det -> 'the' | 'a' | 'my'
    N -> 'man' | 'dog' | 'cat' | 'telescope' | 'park' | 'fish' | 'elephant'
    V -> 'saw' | 'ate' | 'walked' | 'chased'
    P -> 'in' | 'on' | 'by' | 'with'
""")

print("Grammar rules:")
for production in grammar.productions():
    print(f"  {production}")

# 2. Sentence parsing with Recursive Descent Parser
print("\n2. Sentence Parsing with Recursive Descent Parser...")
sentences = [
    "I saw the man with the telescope",
    "the dog chased the cat in the park",
    "a fish ate the elephant",
    "my cat walked by the dog"
]

rd_parser = RecursiveDescentParser(grammar)

for sentence in sentences:
    print(f"\nParsing: '{sentence}'")
    tokens = word_tokenize(sentence)
    trees = list(rd_parser.parse(tokens))

    if trees:
        for i, tree in enumerate(trees):
            print(f"Parse {i+1}:")
            print(tree)
            # Display the tree
            tree.pretty_print()
    else:
        print("No valid parse found")

# 3. Chart Parsing
print("\n3. Chart Parsing...")
chart_parser = ChartParser(grammar)

for sentence in sentences[:2]:  # Just parse first two for demo
    print(f"\nChart parsing: '{sentence}'")
    tokens = word_tokenize(sentence)
    trees = list(chart_parser.parse(tokens))

    if trees:
        for i, tree in enumerate(trees):
            print(f"Parse {i+1}:")
            print(tree)
    else:
        print("No valid parse found")

# 4. Probabilistic Context-Free Grammar
print("\n4. Probabilistic Context-Free Grammar...")
pcfg_grammar = nltk.PCFG.fromstring("""
    S -> NP VP [1.0]
    NP -> Det N [0.5] | 'I' [0.3] | NP PP [0.2]
    VP -> V NP [0.7] | VP PP [0.3]
    PP -> P NP [1.0]
    Det -> 'the' [0.8] | 'a' [0.2]
    N -> 'man' [0.2] | 'dog' [0.2] | 'cat' [0.2] | 'telescope' [0.2] | 'park' [0.2]
    V -> 'saw' [0.4] | 'ate' [0.3] | 'walked' [0.3]
    P -> 'in' [0.4] | 'on' [0.3] | 'by' [0.3]
""")

print("PCFG rules:")
for production in pcfg_grammar.productions():
    print(f"  {production}")

# 5. Viterbi Parsing
print("\n5. Viterbi Parsing...")
viterbi_parser = nltk.ViterbiParser(pcfg_grammar)

for sentence in sentences[:2]:  # Just parse first two for demo
    print(f"\nViterbi parsing: '{sentence}'")
    tokens = word_tokenize(sentence)
    trees = list(viterbi_parser.parse(tokens))

    if trees:
        for i, tree in enumerate(trees):
            print(f"Parse {i+1} (probability: {tree.prob():.6f}):")
            print(tree)
    else:
        print("No valid parse found")

# 6. Dependency Parsing
print("\n6. Dependency Parsing...")
try:
    nltk.data.find('tokenizers/punkt')
    nltk.data.find('taggers/averaged_perceptron_tagger')
    nltk.data.find('chunkers/maxent_ne_chunker')
    nltk.data.find('corpora/dependency_treebank')
except LookupError:
    nltk.download('punkt')
    nltk.download('averaged_perceptron_tagger')
    nltk.download('maxent_ne_chunker')
    nltk.download('dependency_treebank')

# Load dependency grammar
dependency_grammar = nltk.DependencyGrammar.fromstring("""
    'saw' -> 'I' | 'man' | 'telescope'
    'man' -> 'the' | 'with'
    'with' -> 'telescope'
    'telescope' -> 'the'
""")

print("Dependency grammar:")
print(dependency_grammar)

# Create dependency parser
dp = nltk.ProjectiveDependencyParser(dependency_grammar)

# Parse sentence
sentence = "I saw the man with the telescope"
tokens = word_tokenize(sentence)
trees = list(dp.parse(tokens))

print(f"\nDependency parsing for: '{sentence}'")
if trees:
    for i, tree in enumerate(trees):
        print(f"Dependency tree {i+1}:")
        print(tree)
        tree.pretty_print()
else:
    print("No valid dependency parse found")

# 7. Visualizing parse trees
print("\n7. Visualizing Parse Trees...")
# Create a simple parse tree for visualization
simple_grammar = CFG.fromstring("""
    S -> NP VP
    NP -> Det N
    VP -> V NP
    Det -> 'the'
    N -> 'dog' | 'cat'
    V -> 'chased'
""")

simple_parser = ChartParser(simple_grammar)
simple_sentence = "the dog chased the cat"
simple_tokens = word_tokenize(simple_sentence)
simple_trees = list(simple_parser.parse(simple_tokens))

if simple_trees:
    tree = simple_trees[0]
    print("Parse tree structure:")
    print(tree)

    # Display the tree
    tree.pretty_print()

    # Create a graphical visualization
    plt.figure(figsize=(12, 8))
    tree.draw()
    plt.title('Parse Tree Visualization')
    plt.show()

# Memory optimization example with NLTK
import nltk
from nltk.corpus import brown
from nltk.probability import FreqDist
import sys
import time
import matplotlib.pyplot as plt

print("\nMemory Optimization Example...")

# 1. Memory usage comparison
print("1. Memory Usage Comparison...")

def get_memory_usage():
    """Get current memory usage in MB"""
    import psutil
    process = psutil.Process()
    return process.memory_info().rss / 1024 / 1024

# Method 1: Load all words at once
print("\nMethod 1: Load all words at once")
start_time = time.time()
start_mem = get_memory_usage()

all_words = brown.words()
fdist1 = FreqDist(all_words)

end_mem = get_memory_usage()
end_time = time.time()

print(f"Words loaded: {len(all_words):,}")
print(f"Memory used: {end_mem - start_mem:.2f} MB")
print(f"Time taken: {end_time - start_time:.4f} seconds")

# Method 2: Process in chunks
print("\nMethod 2: Process in chunks")
start_time = time.time()
start_mem = get_memory_usage()

fdist2 = FreqDist()
chunk_size = 10000
file_count = len(brown.fileids())

for i, fileid in enumerate(brown.fileids()):
    words = brown.words(fileid)
    fdist2.update(words)

    # Print progress
    if (i + 1) % 100 == 0:
        print(f"Processed {i + 1}/{file_count} files", end='\r')

end_mem = get_memory_usage()
end_time = time.time()

print(f"\nWords processed: {fdist2.N():,}")
print(f"Memory used: {end_mem - start_mem:.2f} MB")
print(f"Time taken: {end_time - start_time:.4f} seconds")

# Method 3: Generator-based processing
print("\nMethod 3: Generator-based processing")
start_time = time.time()
start_mem = get_memory_usage()

def word_generator():
    for fileid in brown.fileids():
        for word in brown.words(fileid):
            yield word

fdist3 = FreqDist(word_generator())

end_mem = get_memory_usage()
end_time = time.time()

print(f"Words processed: {fdist3.N():,}")
print(f"Memory used: {end_mem - start_mem:.2f} MB")
print(f"Time taken: {end_time - start_time:.4f} seconds")

# Compare results
print("\nComparing results...")
print(f"Method 1 vs Method 2: {fdist1.N() == fdist2.N()}")
print(f"Method 1 vs Method 3: {fdist1.N() == fdist3.N()}")

print("\nTop 10 words comparison:")
print("Method 1:", fdist1.most_common(10))
print("Method 2:", fdist2.most_common(10))
print("Method 3:", fdist3.most_common(10))

# 2. Memory-efficient data structures
print("\n2. Memory-efficient Data Structures...")

# Compare memory usage of different data structures
import numpy as np
from collections import Counter

# Create sample data
sample_words = brown.words()[:100000]

# Method 1: Python list
start_mem = get_memory_usage()
word_list = list(sample_words)
list_mem = get_memory_usage() - start_mem
print(f"Python list memory: {list_mem:.2f} MB")

# Method 2: Python set
start_mem = get_memory_usage()
word_set = set(sample_words)
set_mem = get_memory_usage() - start_mem
print(f"Python set memory: {set_mem:.2f} MB")

# Method 3: NLTK FreqDist
start_mem = get_memory_usage()
fdist = FreqDist(sample_words)
fdist_mem = get_memory_usage() - start_mem
print(f"NLTK FreqDist memory: {fdist_mem:.2f} MB")

# Method 4: Python Counter
start_mem = get_memory_usage()
counter = Counter(sample_words)
counter_mem = get_memory_usage() - start_mem
print(f"Python Counter memory: {counter_mem:.2f} MB")

# Method 5: NumPy array
start_mem = get_memory_usage()
# Convert words to unique IDs
unique_words = list(set(sample_words))
word_to_id = {word: i for i, word in enumerate(unique_words)}
word_ids = np.array([word_to_id[word] for word in sample_words])
numpy_mem = get_memory_usage() - start_mem
print(f"NumPy array memory: {numpy_mem:.2f} MB")

# Plot memory usage comparison
methods = ['List', 'Set', 'FreqDist', 'Counter', 'NumPy']
memory_usage = [list_mem, set_mem, fdist_mem, counter_mem, numpy_mem]

plt.figure(figsize=(10, 6))
plt.bar(methods, memory_usage, color=['blue', 'green', 'red', 'purple', 'orange'])
plt.title('Memory Usage Comparison of Data Structures')
plt.ylabel('Memory Usage (MB)')
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()

# 3. Caching frequent operations
print("\n3. Caching Frequent Operations...")
from functools import lru_cache

# Without caching
def slow_pos_tag(text):
    """Simulate a slow POS tagging operation"""
    time.sleep(0.1)  # Simulate processing time
    return nltk.pos_tag(word_tokenize(text))

# With caching
@lru_cache(maxsize=100)
def cached_pos_tag(text):
    """Cached version of POS tagging"""
    return nltk.pos_tag(word_tokenize(text))

# Test performance
texts = [
    "The quick brown fox jumps over the lazy dog.",
    "Natural language processing is fascinating.",
    "Machine learning and NLP work well together.",
    "The quick brown fox jumps over the lazy dog.",  # Duplicate
    "Natural language processing is amazing."        # Similar but not identical
]

print("Testing without caching...")
start_time = time.time()
for text in texts:
    result = slow_pos_tag(text)
print(f"Time without caching: {time.time() - start_time:.4f} seconds")

print("\nTesting with caching...")
start_time = time.time()
for text in texts:
    result = cached_pos_tag(text)
print(f"Time with caching: {time.time() - start_time:.4f} seconds")

print(f"\nCache info: {cached_pos_tag.cache_info()}")

# Parallel processing example with NLTK
import nltk
from nltk.corpus import brown
from nltk.probability import FreqDist
import time
import multiprocessing
from multiprocessing import Pool
import matplotlib.pyplot as plt

print("\nParallel Processing Example...")

# 1. Sequential processing
print("1. Sequential Processing...")
def process_file(fileid):
    """Process a single file and return word frequency distribution"""
    words = brown.words(fileid)
    return FreqDist(words)

# Get all file IDs
fileids = brown.fileids()
print(f"Processing {len(fileids)} files...")

start_time = time.time()
sequential_fdist = FreqDist()

for fileid in fileids:
    fdist = process_file(fileid)
    sequential_fdist.update(fdist)

sequential_time = time.time() - start_time
print(f"Sequential processing time: {sequential_time:.4f} seconds")
print(f"Total words processed: {sequential_fdist.N():,}")

# 2. Parallel processing
print("\n2. Parallel Processing...")
def parallel_process_file(args):
    """Wrapper function for parallel processing"""
    fileid, category = args
    words = brown.words(fileid)
    return FreqDist(words)

# Prepare arguments for parallel processing
args = [(fileid, brown.categories(fileid)[0]) for fileid in fileids]

# Determine number of processes
num_processes = multiprocessing.cpu_count()
print(f"Using {num_processes} processes...")

start_time = time.time()
with Pool(processes=num_processes) as pool:
    results = pool.map(parallel_process_file, args)

parallel_fdist = FreqDist()
for fdist in results:
    parallel_fdist.update(fdist)

parallel_time = time.time() - start_time
print(f"Parallel processing time: {parallel_time:.4f} seconds")
print(f"Total words processed: {parallel_fdist.N():,}")

# 3. Performance comparison
print("\n3. Performance Comparison...")
print(f"Sequential time: {sequential_time:.4f} seconds")
print(f"Parallel time: {parallel_time:.4f} seconds")
print(f"Speedup: {sequential_time / parallel_time:.2f}x")

# Verify results are the same
print(f"\nResults match: {sequential_fdist.N() == parallel_fdist.N()}")
print(f"Top 10 words match: {sequential_fdist.most_common(10) == parallel_fdist.most_common(10)}")

# Plot performance comparison
plt.figure(figsize=(10, 6))
plt.bar(['Sequential', 'Parallel'], [sequential_time, parallel_time], color=['blue', 'green'])
plt.title('Processing Time Comparison')
plt.ylabel('Time (seconds)')
plt.show()

# 4. Parallel processing with different chunk sizes
print("\n4. Parallel Processing with Different Chunk Sizes...")
chunk_sizes = [1, 10, 50, 100, 200]
times = []

for chunk_size in chunk_sizes:
    print(f"Testing chunk size: {chunk_size}")
    start_time = time.time()

    # Split fileids into chunks
    chunks = [fileids[i:i + chunk_size] for i in range(0, len(fileids), chunk_size)]

    # Process each chunk in parallel
    with Pool(processes=num_processes) as pool:
        # Process each chunk (multiple files at once)
        chunk_results = pool.map(process_chunk, chunks)

    # Combine results
    chunk_fdist = FreqDist()
    for fdist in chunk_results:
        chunk_fdist.update(fdist)

    chunk_time = time.time() - start_time
    times.append(chunk_time)
    print(f"Time with chunk size {chunk_size}: {chunk_time:.4f} seconds")

# Plot chunk size vs performance
plt.figure(figsize=(10, 6))
plt.plot(chunk_sizes, times, marker='o')
plt.title('Chunk Size vs Processing Time')
plt.xlabel('Chunk Size (files per task)')
plt.ylabel('Time (seconds)')
plt.grid(True)
plt.show()

def process_chunk(chunk):
    """Process a chunk of files"""
    chunk_fdist = FreqDist()
    for fileid in chunk:
        words = brown.words(fileid)
        chunk_fdist.update(words)
    return chunk_fdist

# 5. Parallel processing with category analysis
print("\n5. Parallel Processing with Category Analysis...")
def process_category(category):
    """Process all files in a category"""
    fileids = brown.fileids(category)
    category_fdist = FreqDist()

    for fileid in fileids:
        words = brown.words(fileid)
        category_fdist.update(words)

    return category, category_fdist

# Get all categories
categories = brown.categories()
print(f"Processing {len(categories)} categories...")

start_time = time.time()
with Pool(processes=num_processes) as pool:
    category_results = pool.map(process_category, categories)

category_fdist = {category: fdist for category, fdist in category_results}
parallel_category_time = time.time() - start_time

print(f"Parallel category processing time: {parallel_category_time:.4f} seconds")

# Compare with sequential
start_time = time.time()
sequential_category_fdist = {}
for category in categories:
    fileids = brown.fileids(category)
    category_fdist = FreqDist()

    for fileid in fileids:
        words = brown.words(fileid)
        category_fdist.update(words)

    sequential_category_fdist[category] = category_fdist

sequential_category_time = time.time() - start_time
print(f"Sequential category processing time: {sequential_category_time:.4f} seconds")
print(f"Speedup: {sequential_category_time / parallel_category_time:.2f}x")

# Compare results
print("\nCategory results comparison:")
for category in categories:
    match = (sequential_category_fdist[category].N() ==
             category_fdist[category].N())
    print(f"{category:15}: {match}")