A step -by -step information to construct a development search instrument with Python: internet scraping, NLP (emotions evaluation and matters modeling) and Phrase Cloud visualization

import requests
from bs4 import BeautifulSoup


# Checklist of URLs to scrape
urls = ("https://en.wikipedia.org/wiki/Natural_language_processing",
        "https://en.wikipedia.org/wiki/Machine_learning")  


collected_texts = ()  # to retailer textual content from every web page


for url in urls:
    response = requests.get(url, headers={"Consumer-Agent": "Mozilla/5.0"})
    if response.status_code == 200:
        soup = BeautifulSoup(response.textual content, 'html.parser')
        # Extract all paragraph textual content
        paragraphs = (p.get_text() for p in soup.find_all('p'))
        page_text = " ".be part of(paragraphs)
        collected_texts.append(page_text.strip())
    else:
        print(f"Did not retrieve {url}")

import re
import nltk
nltk.obtain('stopwords')
from nltk.corpus import stopwords


stop_words = set(stopwords.phrases('english'))


cleaned_texts = ()
for textual content in collected_texts:
    # Take away non-alphabetical characters and decrease the textual content
    textual content = re.sub(r'(^A-Za-zs)', ' ', textual content).decrease()
    # Take away stopwords
    phrases = (w for w in textual content.break up() if w not in stop_words)
    cleaned_texts.append(" ".be part of(phrases))

from collections import Counter


# Mix all texts into one if analyzing general tendencies:
all_text = " ".be part of(cleaned_texts)
word_counts = Counter(all_text.break up())
common_words = word_counts.most_common(10)  # high 10 frequent phrases
print("High 10 key phrases:", common_words)

!pip set up textblob
from textblob import TextBlob


for i, textual content in enumerate(cleaned_texts, 1):
    polarity = TextBlob(textual content).sentiment.polarity
    if polarity > 0.1:
        sentiment = "Constructive 😀"
    elif polarity < -0.1:
        sentiment = "Adverse 🙁"
    else:
        sentiment = "Impartial 😐"
    print(f"Doc {i} Sentiment: {sentiment} (polarity={polarity:.2f})")

from sklearn.feature_extraction.textual content import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation


# Regulate these parameters
vectorizer = CountVectorizer(max_df=1.0, min_df=1, stop_words="english")
doc_term_matrix = vectorizer.fit_transform(cleaned_texts)


# Match LDA to search out matters (for example, 3 matters)
lda = LatentDirichletAllocation(n_components=3, random_state=42)
lda.match(doc_term_matrix)


feature_names = vectorizer.get_feature_names_out()


for idx, subject in enumerate(lda.components_):
    print(f"Matter {idx + 1}: ", (vectorizer.get_feature_names_out()(i) for i in subject.argsort()(:-11:-1)))

# Assuming you have got your textual content knowledge saved in combined_text
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import nltk
from nltk.corpus import stopwords
import re


nltk.obtain('stopwords')
stop_words = set(stopwords.phrases('english'))


# Preprocess and clear the textual content:
cleaned_texts = ()
for textual content in collected_texts:
    textual content = re.sub(r'(^A-Za-zs)', ' ', textual content).decrease()
    phrases = (w for w in textual content.break up() if w not in stop_words)
    cleaned_texts.append(" ".be part of(phrases))


# Generate mixed textual content
combined_text = " ".be part of(cleaned_texts)


# Generate the phrase cloud
wordcloud = WordCloud(width=800, peak=400, background_color="white", colormap='viridis').generate(combined_text)


# Show the phrase cloud
plt.determine(figsize=(10, 6))  # <-- corrected numeric dimensions
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.title("Phrase Cloud of Scraped Textual content", fontsize=16)
plt.present()