#set document(title: "6.5 Other Machine Learning Techniques", author: "OpenStax / XYZ Homework")
#set page(width: 8.5in, height: auto, margin: 1in)
#import "@preview/cetz:0.5.2"
#set text(font: ("STIX Two Text", "Libertinus Serif", "New Computer Modern"), size: 10.5pt, lang: "en")
#show math.equation: set text(font: ("STIX Two Math", "New Computer Modern Math"))
#set par(justify: true, leading: 0.62em, spacing: 0.9em)
#set enum(spacing: 1.1em)   // room between list items so tall inline fractions don't collide
#set list(spacing: 1.1em)
#set table(stroke: 0.5pt + rgb("#c7ccd3"))

#let BLUE = rgb("#183B6F")     // brand navy — section bars + example/solution labels (white on navy 11.09:1)
#let ORANGE = rgb("#A94509")   // brand primary-700 — AA-safe deep orange for TEXT (5.93:1 on white; raw brand #F37021 is 2.94:1 and must never carry text)
#let RED = rgb("#DC2626")      // brand error-600
#let GREEN = rgb("#059669")    // brand success-600 (decoration only; small green text uses green-text #007942)

#show heading.where(level: 1): it => block(width: 100%, above: 0pt, below: 16pt,
  fill: gradient.linear(BLUE, rgb("#2C5AA0")), inset: (x: 14pt, y: 12pt), radius: 3pt,
  text(fill: white, weight: "bold", size: 19pt, it.body))
#show heading.where(level: 2): it => block(width: 100%, above: 18pt, below: 10pt,
  fill: BLUE, inset: (x: 10pt, y: 6pt), radius: 2pt,
  text(fill: white, weight: "bold", size: 12pt, it.body))
#show heading.where(level: 3): it => text(fill: ORANGE, weight: "bold", size: 12.5pt, it.body)
#show heading.where(level: 4): it => text(fill: BLUE, weight: "bold", size: 10.5pt, it.body)

#let examplebox(label, title, body) = block(width: 100%, breakable: true,
  fill: rgb("#EFF1F5"), stroke: 0.5pt + rgb("#CFDDF0"), radius: 4pt, inset: 10pt,
  above: 12pt, below: 12pt)[
  #block(below: 6pt)[#box(fill: BLUE, inset: (x: 6pt, y: 2pt), radius: 2pt,
    text(fill: white, weight: "bold", size: 8.5pt, label)) #h(0.4em) #strong[#title]]
  #body]

// rail = decorative left rule (raw brand token); labelcolor = AA-safe label text shade
#let notebox(label, rail, labelcolor, tint, body) = block(width: 100%, breakable: true, fill: tint,
    stroke: (left: 3pt + rail), inset: (left: 10pt, rest: 8pt), radius: (right: 4pt),
    above: 11pt, below: 11pt)[
    #text(fill: labelcolor, weight: "bold", size: 7.5pt, tracking: 0.5pt)[#upper(label)]
    #linebreak() #body]

#let solutionbox(body) = block(above: 4pt, below: 8pt)[
  #text(fill: BLUE, weight: "bold", size: 8.5pt)[Solution] #linebreak() #body]

#let figph(msg) = block(width: 100%, height: 60pt, fill: rgb("#f6f7f9"),
  stroke: (paint: rgb("#c7ccd3"), dash: "dashed"), radius: 4pt, inset: 10pt)[
  #align(center + horizon, text(fill: rgb("#889"), style: "italic", size: 9pt, msg))]

// Standardize inlined figure sizes: measure the natural CeTZ canvas, then scale to a
// consistent envelope (aspect-aware; see build_typst.py FIG_* constants). Unlike the
// print preamble, dimensions are FLOORED: in an editor a user can trim a figure to a
// degenerate 1-D shape (a bare line), and w/h or tw/w would then divide by zero.
#let _STD_W = 3.5
#let _WIDE_W = 5.6
#let _MAX_H = 3.4
#let _ASPECT_WIDE = 2.2
#let _UPSCALE_MAX = 1.15
#let stdfig(body) = context {
  let m = measure(body)
  let w = calc.max(m.width / 1in, 0.01)
  let h = calc.max(m.height / 1in, 0.01)
  let tw = if w / h > _ASPECT_WIDE { _WIDE_W } else { _STD_W }
  let s = calc.min(tw / w, _MAX_H / h, _UPSCALE_MAX)
  align(center, box(scale(x: s * 100%, y: s * 100%, reflow: true, body)))
}

#show figure: set block(breakable: false)
#set figure(gap: 8pt)
#show figure.caption: set text(size: 8.5pt, fill: rgb("#555"))


  
== 6.5#h(0.6em)Other Machine Learning Techniques


  
    
=== Learning Outcomes


    By the end of this section, you should be able to:


    - Discuss the concept of a random forest as a bootstrapping method for decision trees.
- Create a random forest model and use it to classify data.
- Define conditional probability and explain prior probabilities for training datasets.
- Produce a (multinomial) naïve Bayes classifier and use it to classify data.
- Discuss the concept of a Gaussian naïve Bayes classifier.
- Describe some methods for working with big data efficiently and effectively.


  
  So far, we have encountered a number of different machine learning algorithms suited to particular tasks. Naturally, the realm of machine learning extends well beyond this small survey of methods. This section describes several more machine learning techniques, though in less detail. Methods such as random forests, naïve Bayes classifiers, and a refinement of the latter called Gaussian naïve Bayes, offer distinctive approaches to solving complex problems. Additionally, this section provides insights into strategies for handling vast datasets, presenting a comprehensive survey of methods tailored for the unique challenges posed by Big Data.


  
    
=== Random Forests


    In Decision Trees, we constructed decision trees and discussed some methods, such as pruning, that would improve the reliability of predictions of the decision tree. For each classification problem, one tree is created that serves as the model for all subsequent purposes. But no matter how much care is taken in creating and pruning a decision tree, at the end of the day, it is a single model that may have some bias built into its very structure due to variations in the training set. If we used a different training set, then our decision tree may have come out rather differently.


    In the real world, obtaining a sufficient amount of data to train a machine learning model may be difficult, time-consuming, and expensive. It may not be practical to go out and find additional training sets just to create and evaluate a variety of decision trees. A #strong[random forest], a classification algorithm that uses multiple decision trees, serves as a way to get around this problem. Similar to what we did for linear regressions in Machine Learning in Regression Analysis, the technique of #emph[bootstrap aggregating] is employed. In this context, #strong[bootstrap aggregating (bagging)] involves resampling from the same testing dataset multiple times, creating a new decision tree each time. The individual decision trees that make up a random forest model are called #strong[weak learners]. To increase the diversity of the weak learners, there is random feature selection, meaning that each tree is built using a randomly selected subset of the feature variables. For example, if we are trying to predict the presence of heart disease in a patient using age, weight, and cholesterol levels, then a random forest model will consist of various decision trees, some of which may use only age and weight, while others use age and cholesterol level, and still others may use weight alone. When classifying new data, the final prediction of the random forest is determined by “majority vote” among the individual decision trees. What’s more, the fact that individual decision trees use only a subset of the features means that the importance of each feature can be inferred by the accuracy of decision trees that utilize that feature.


    Because of the complexity of this method, random forests will be illustrated by way of the following Python code block.


    Suppose you are building a model that predicts daily temperatures based on such factors as the temperature yesterday and the day before as well forecasts from multiple sources, using the dataset #link("https://openstax.org/r/temps")[#strong[temps.csv]]. Since there are a lot of input features, it will not be possible to visualize the data. Moreover, some features are likely not to contribute much to the predictions. How do we sort everything out? A random forest model may be just the right tool. The Python library #link("https://openstax.org/r/scikit1")[#strong[sklearn]] has a module called #strong[sklearn.ensemble] that is used to create the random forest model.


    #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
\# Import libraries
      import pandas as pd  \#\# for dataset management
      from sklearn.model\_selection import train\_test\_split
      import sklearn.ensemble as ens
      
      \# Read input file
      features = pd.read\_csv('temps.csv').dropna()
      
      \# Use 'actual' as the response variable
      labels = features\['actual'\]
      
      \# Convert text data into numerical values
      \# This is called "one-hot" encoding
      features = pd.get\_dummies(features)
      
      \# The other columns are used as features
      features = features.drop('actual', axis=1)
      feature\_list = list(features.columns)
      
      \# Split data into training and testing sets
      train\_features, test\_features, train\_labels, test\_labels = train\_test\_split(features, labels, test\_size=0.25)
      
      \# Create random forest model
      rf = ens.RandomForestRegressor(n\_estimators=1000)
      rf.fit(train\_features, train\_labels)
    
      The resulting output will look like this:


      #figure(figph[Output from Python code that says “RandomForestRegressor” on the top line with a light blue background and “RandomForestRegressor(n\_estimators=1000) on the bottom line.”], alt: "Output from Python code that says “RandomForestRegressor” on the top line with a light blue background and “RandomForestRegressor(n_estimators=1000) on the bottom line.”", caption: none)
]


    The preceding Python code reads in the dataset #link("https://openstax.org/r/temps")[#strong[temps.csv]], identifying the column “actual” (actual temperatures each day) as the variable to be predicted. All the other columns are assumed to be features. Since the “week” column contains categorical (text) data, it needs to be converted to numerical data before any model can be set up. The method of one-hot encoding does the trick here. In #strong[one-hot encoding], each category is mapped onto a vector containing a single 1 corresponding to that category while all other categories are set to 0. In our example, each day of the week maps to a seven-dimensional vector as follows:


    - Monday:		\[1, 0, 0, 0, 0, 0, 0\]
- Tuesday:		\[0, 1, 0, 0, 0, 0, 0\]
- Wednesday: 	\[0, 0, 1, 0, 0, 0, 0\]
- Thursday:		\[0, 0, 0, 1, 0, 0, 0\]
- Friday: 		\[0, 0, 0, 0, 1, 0, 0\]
- Saturday: 	\[0, 0, 0, 0, 0, 1, 0\]
- Sunday: 		\[0, 0, 0, 0, 0, 0, 1\]


    The dataset is split into training (75%) and testing (25%) sets, and the random forest model is trained on the training set, as seen in this Python code block:


    #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
import statistics as stats \# to compute the mean

      \# Get predictions and compute average error
      predictions = rf.predict(test\_features)
      errors = abs(predictions - test\_labels)
      round(np.mean(errors),2)
    
      The resulting output will look like this:


      3.91
]


    With a mean absolute error of only 3.91, predicted temperatures are only off by about 4°F on average, so the random forest seems to do a good job with the test set, given that it is notoriously difficult to predict weather data since it typically shows high variance. Now let’s find out which features were most important in making predictions, which is stored in rf.feature\_importances\_. Note the number of Python commands used solely for sorting and formatting the results, which can be safely ignored in this snippet of code.


    #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
\# Find the importance of each feature
    importances = list(rf.feature\_importances\_)
    feature\_importances = \[(feature, round(importance,2)) for feature, importance in zip(feature\_list, importances)\]
    feature\_importances = sorted(feature\_importances, key=lambda x: x\[1\], reverse=True)
    \[print('Variable: {:20} Importance: {}'.format(\*pair)) for pair in feature\_importances\];
    
      The resulting output will look like this:


      Variable: temp\_1               Importance: 0.62 #linebreak() Variable: average              Importance: 0.19 #linebreak() Variable: forecast\_acc         Importance: 0.06 #linebreak() Variable: forecast\_noaa        Importance: 0.04 #linebreak() Variable: day                  Importance: 0.02 #linebreak() Variable: temp\_2               Importance: 0.02 #linebreak() Variable: forecast\_under       Importance: 0.02 #linebreak() Variable: friend               Importance: 0.02 #linebreak() Variable: month                Importance: 0.01 #linebreak() Variable: year                 Importance: 0.0 #linebreak() Variable: week\_Fri             Importance: 0.0 #linebreak() Variable: week\_Mon             Importance: 0.0 #linebreak() Variable: week\_Sat             Importance: 0.0 #linebreak() Variable: week\_Sun             Importance: 0.0 #linebreak() Variable: week\_Thurs           Importance: 0.0 #linebreak() Variable: week\_Tues            Importance: 0.0 #linebreak() Variable: week\_Wed             Importance: 0.0
]


    As we can see, the single best predictor of daily temperatures is the temperature on the previous day (“temp\_1”), followed by the average temperature on this day in previous years (“average”). The forecasts from NOAA and ACC showed only minor importance, while all other feature variables were relatively useless in predicting current temperatures. It should come as no surprise that the day of the week (Monday, Tuesday, etc.) played no significant part in predicting temperatures.


    #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
#emph[Using Random Forests for Facial Recognition]


      As noted in this chapter’s introduction, facial recognition is an important and widely used application of machine learning—and an example of a classification task. #strong[Facial recognition] involves categorizing or labeling images of faces based on the identities of individuals depicted in those images, which is a #emph[multiclass classification task]. (In its simplest form, when the task is to determine whether a given image contains the face of a particular individual or #emph[not], this is considered a#emph[binary classification task].) When the data consist of many images of faces, each containing hundreds or thousands of features, using random forests would be appropriate. Check out this Colab Notebook, written by Michael Beyeler, a neuroscientist at the University of California, Santa Barbara, which steps through the process of setting up and using a random forest model for facial recognition in Python.
]


  
  
    
=== Multinomial Naïve Bayes Classifiers


    One method for classifying data is to look for commonly occurring features that would distinguish one data point from another. For example, imagine you have a vast collection of news articles and you want to automatically categorize them into topics such as sports, politics, business, entertainment, and technology. You may have noticed certain patterns, such as sport articles tend to mention team names and give numerical scores, while articles about politics mention the words “Democrat” and “Republican” fairly often. Business articles might use words such as “outlook” and “downturn” much more often than entertainment or technology articles would. Multinomial naïve Bayes can be applied to classify these articles based on the frequency and distribution of words typically found in each kind of article.


    The multinomial #strong[naïve Bayes classification] algorithm is based on Bayes’ Theorem, making use of prior probabilities and Bayes’ Theorem to predict the class or label of new data. (See Normal Continuous Probability Distributions for more details.) The naïve Bayes classifier algorithm is a powerful tool for working out probabilities of events based on prior knowledge. Recall, the notation #math.equation(block: false, alt: "P open parenthesis A | B close parenthesis")[$P ( A | B )$] stands for the conditional probability that event #math.equation(block: false, alt: "A")[$A$] occurs given that event #math.equation(block: false, alt: "B")[$B$] is known to occur (or has already occurred or is presumed to occur). In the simplest version, Bayes’ Theorem takes the form


    #math.equation(block: true, alt: "P open parenthesis A | B close parenthesis equals the fraction P open parenthesis A close parenthesis P open parenthesis B | A close parenthesis over P open parenthesis B close parenthesis")[$P ( A | B ) = frac(P ( A ) P ( B | A ), P ( B ))$]
    Here, we regard #math.equation(block: false, alt: "A")[$A$] as an event that we would like to predict and #math.equation(block: false, alt: "B")[$B$] as information that has been given to us (perhaps as a feature of a dataset). The conditional probability #math.equation(block: false, alt: "P open parenthesis B | A close parenthesis")[$P ( B | A )$] represents the known likelihood of seeing event #math.equation(block: false, alt: "B")[$B$] in the case that #emph[A] occurs, which is typically found using the training data. The value of #math.equation(block: false, alt: "P open parenthesis A close parenthesis")[$P ( A )$] would be computed or estimated from training data as well and is known as the #strong[prior probability].


    #examplebox("Example 1")[][
Three companies supply batteries for a particular model of electric vehicle. Acme Electronics supplies 25% of the batteries, and 2.3% of them are defective. Batteries ‘R’ Us supplies 36% of the batteries, and 1.7% of them are defective. Current Trends supplies the remaining 39% of the batteries, and 2.1% of them are defective. A defective battery is delivered without information about its origin. What is the probability that it came from Acme Electronics?


      
      #solutionbox[
First, we find the probability that a delivered battery is defective. Let #math.equation(block: false, alt: "A")[$A$], #math.equation(block: false, alt: "B")[$B$], and #math.equation(block: false, alt: "C")[$C$] stand for the proportions of batteries from each of the companies, Acme Electronics, Batteries ‘R’ Us, and Current Trends, respectively. Then #math.equation(block: false, alt: "P open parenthesis A close parenthesis equals 0.25 , P open parenthesis B close parenthesis equals 0.36 , P open parenthesis C close parenthesis equals 0.39")[$P ( A ) = 0.25 , P ( B ) = 0.36 , P ( C ) = 0.39$]. We also have the probabilities of receiving a defective battery (event #math.equation(block: false, alt: "d")[$d$]) from each company, #math.equation(block: false, alt: "P open parenthesis d | A close parenthesis equals 0.023 , P open parenthesis d | B close parenthesis equals 0.017 , P open parenthesis d | C close parenthesis equals 0.021")[$P ( d | A ) = 0.023 , P ( d | B ) = 0.017 , P ( d | C ) = 0.021$]. Now, the probability of receiving a defective battery from any of the companies is the sum


        #math.equation(block: true, alt: "P open parenthesis d close parenthesis, equals, P open parenthesis A close parenthesis P open parenthesis d | A close parenthesis plus P open parenthesis B close parenthesis P open parenthesis d | B close parenthesis plus P open parenthesis C close parenthesis P open parenthesis d | C close parenthesis; equals, open parenthesis 0.25 close parenthesis open parenthesis 0.023 close parenthesis plus open parenthesis 0.36 close parenthesis open parenthesis 0.017 close parenthesis plus open parenthesis 0.39 close parenthesis open parenthesis 0.021 close parenthesis equals 0.02")[$P ( d ) & = & P ( A ) P ( d | A ) + P ( B ) P ( d | B ) + P ( C ) P ( d | C ) \  & = & ( 0.25 ) ( 0.023 ) + ( 0.36 ) ( 0.017 ) + ( 0.39 ) ( 0.021 ) = 0.02$]
        Using Bayes’ Theorem, the probability that the defective battery came from Acme is:


        #math.equation(block: true, alt: "P open parenthesis A | d close parenthesis equals the fraction P open parenthesis A close parenthesis P open parenthesis d | A close parenthesis over P open parenthesis d close parenthesis equals the fraction open parenthesis 0.25 close parenthesis open parenthesis 0.023 close parenthesis over 0.02 equals 0.29 equals 29 %")[$P ( A | d ) = frac(P ( A ) P ( d | A ), P ( d )) = frac(( 0.25 ) ( 0.023 ), 0.02) = 0.29 = 29 %$]
]
]


    In practice, there are often a multitude of features, #math.equation(block: false, alt: "B sub 1 , B sub 2 , B sub 3 , and so on B sub n")[$B_(1) , B_(2) , B_(3) , … B_(n)$], where each feature could be the event that a particular word occurs in a message, and we would like to classify the message into some type or another based on those events. In other words, we would like to know something about #math.equation(block: false, alt: "P open parenthesis A | B sub 1 , B sub 2 , B sub 3 , and so on , B sub n close parenthesis")[$P ( A | B_(1) , B_(2) , B_(3) , … , B_(n) )$], the probability of A occurring if we know that events #math.equation(block: false, alt: "B sub 1 , B sub 2 , B sub 3 , and so on B sub n")[$B_(1) , B_(2) , B_(3) , … B_(n)$] occurred. In fact, we are just interested in distinguishing event #math.equation(block: false, alt: "A")[$A$] from #math.equation(block: false, alt: "A ′")[$A ′$] rather than precise computations of probabilities, and so we will do simple comparisons to find out this information. The process is best illustrated by example.


    #examplebox("Example 2")[][
A survey of 1,000 news articles was conducted. It is known that 600 were authentic articles and 400 were fake. In the real articles, 432 contained the word #emph[today], 87 contained the word #emph[disaster], and 303 contained the word #emph[police]. In the fake articles, 124 contained the word #emph[today], 320 contained the word #emph[disaster], and 230 contained the word #emph[police]. Predict whether a new article is real or fake if it (a) contains the word “disaster” or (b) contains the words “today” and “police.”


      
      #solutionbox[
First, find the prior probabilities that a news article is real or fake based on the proportion of such articles in the training set. #math.equation(block: false, alt: "P (Real) equals the fraction 600 over 1,000 equals 0.6")[$P "(Real)" #h(0.2em) = frac(600, 1","000) = 0.6$] and #math.equation(block: false, alt: "P (Fake) equals the fraction 400 over 1000 equals 0.4")[$P "(Fake)" #h(0.2em) = frac(400, 1000) = 0.4$]. Next, using the word counts found in the training data, find the conditional probabilities:


        #math.equation(block: true, alt: "P (today|Real) equals the fraction 432 over 600 equals 0.72")[$P "(today|Real)" = frac(432, 600) = 0.72$], #math.equation(block: true, alt: "P (disaster|Real) equals the fraction 87 over 600 equals 0.145")[$P "(disaster|Real)" #h(0.2em) = frac(87, 600) = 0.145$], #math.equation(block: true, alt: "P (police|Real) equals the fraction 303 over 600 equals 0.505")[$P "(police|Real)" #h(0.2em) = frac(303, 600) = 0.505$]


        #math.equation(block: true, alt: "P (today|Fake) equals the fraction 124 over 400 equals 0.31")[$P "(today|Fake)" #h(0.2em) = frac(124, 400) = 0.31$], #math.equation(block: true, alt: "P (disaster|Fake) equals the fraction 320 over 400 equals 0.8")[$P "(disaster|Fake)" #h(0.2em) = frac(320, 400) = 0.8$], #math.equation(block: true, alt: "P (disaster|Fake) equals the fraction 230 over 400 equals 0.575")[$P "(disaster|Fake)" #h(0.2em) = frac(230, 400) = 0.575$]


        Now we will only use the numerator of Bayes’ formula, which is proportional to the exact probability. This will produce scores that we can compare. For part (a):


        #math.equation(block: false, alt: "P open parenthesis Real|disaster close parenthesis")[$P ( "Real|disaster" #h(0.2em) )$] score: #math.equation(block: false, alt: "P (Real) P (disaster|Real) equals open parenthesis 0.6 close parenthesis open parenthesis 0.145 close parenthesis equals 0.087")[$P "(Real)" P "(disaster|Real)" #h(0.2em) = ( 0.6 ) ( 0.145 ) = 0.087$]


        #math.equation(block: false, alt: "P (Fake|disaster)")[$P "(Fake|disaster) "$] score: #math.equation(block: false, alt: "P (Fake) P (disaster|Fake) equals open parenthesis 0.4 close parenthesis open parenthesis 0.8 close parenthesis equals 0.32")[$P "(Fake)" P "(disaster|Fake)" #h(0.2em) = ( 0.4 ) ( 0.8 ) = 0.32$]


        Since the score for #emph[Fake] is greater than the score for #emph[Real], we would classify the article as fake.


        For part (b), the process is analogous. Note that probabilities are multiplied together when there are multiple features present.


        #math.equation(block: false, alt: "P (Real|today, police)")[$P "(Real|today, police)"$] score: #math.equation(block: false, alt: "P (Real) P (today | Real) P (police|Real)")[$P "(Real)" P "(today" #h(0.2em) | "Real)" P "(police|Real)"$]


        #math.equation(block: true, alt: "equals open parenthesis 0.6 close parenthesis open parenthesis 0.72 close parenthesis open parenthesis .505 close parenthesis equals 0.22")[$= ( 0.6 ) ( 0.72 ) ( .505 ) = 0.22$]
        #math.equation(block: false, alt: "P (Fake|today, police)")[$P "(Fake|today, police)"$] score: #math.equation(block: false, alt: "P (Fake) P (today|Fake) P (police|Fake)")[$P "(Fake)" P "(today|Fake)" P "(police|Fake)"$]


        #math.equation(block: true, alt: "equals open parenthesis 0.4 close parenthesis open parenthesis 0.31 close parenthesis open parenthesis .575 close parenthesis equals 0.071")[$= ( 0.4 ) ( 0.31 ) ( .575 ) = 0.071$]
        Here the score for #emph[Real] is greater than that of #emph[Fake], so we conclude the article is real.
]
]


    Note: Naïve Bayes classification assumes that all features of the data are independent. While this condition may not necessarily be true in real-world datasets, naïve Bayes may still perform fairly well in real-world situations. Moreover, naïve Bayes is a very simple and efficient algorithm, and so it is often the first method used in a classification problem before more sophisticated models are employed.


  
  
    
=== Gaussian Naïve Bayes for Continuous Probabilities


    #strong[Gaussian naïve Bayes] is a variant of the naïve Bayes classification algorithm that is specifically designed for data with continuous features. Unlike the standard (multinomial) naïve Bayes, which is commonly used to classify text data with discrete features, Gaussian naïve Bayes is suitable for datasets with features that follow a Gaussian (normal) distribution (You may recall the discussion of the normal distribution in Discrete and Continuous Probability Distributions). It's particularly useful for problems involving real-valued, continuous data in which the feature variables are relatively independent.


    In this example, we will work with a large dataset, #link("https://openstax.org/r/cirrhosis")[#strong[cirrhosis.csv]], that has many features, both categorical and numerical. This dataset (along with many more!) is available for free download from Kaggle.com. We will only use four of the numerical columns as features. The response variables is “Status,” which may take the values #math.equation(block: false, alt: "D")[$D$] for death, #math.equation(block: false, alt: "C")[$C$] for censored (meaning that the patient did not die during the observation period), and #emph[CL] for censored due to liver transplantation.


    #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
\# Import libraries
      import pandas as pd  \#\# for dataset management
      from sklearn.model\_selection import train\_test\_split
      from sklearn.naive\_bayes import GaussianNB
      from sklearn.metrics import accuracy\_score, confusion\_matrix, classification\_report
      
      \# Load the dataset
      data = pd.read\_csv('cirrhosis.csv').dropna()
      
      \# Choose feature columns and target labels
      X = data\[\['Bilirubin','Cholesterol','Albumin','Copper'\]\]
      y = data\['Status'\]
      
      \# Split the dataset into training and testing sets (80% training, 20% testing)
      X\_train, X\_test, y\_train, y\_test = train\_test\_split(X, y, test\_size=0.2)
      
      \# Initialize and train the Gaussian naive Bayes classifier
      gnb\_classifier = GaussianNB()
      gnb\_classifier.fit(X\_train, y\_train)
      
      \# Predict labels for the test set
      y\_pred = gnb\_classifier.predict(X\_test)
      
      \# Evaluate the classifier's performance
      accuracy = accuracy\_score(y\_test, y\_pred)
      confusion = confusion\_matrix(y\_test, y\_pred, labels=\['D','C','CL'\])
      report = classification\_report(y\_test, y\_pred)
      
      print(f"Accuracy: {accuracy:.2f}")
      print("Confusion matrix with label order D, C, CL:\\n", confusion)
    
      The resulting output will look like this:


      Accuracy: 0.71 #linebreak() Confusion matrix with label order D, C, CL: #linebreak() \[\[10 14  2\] #linebreak() \[ 0 30  0\] #linebreak() \[ 0  0  0\]\]
]


    The accuracy score of 0.71 means that correct labels were assigned 71% of the time. We can see that 10 deaths (#math.equation(block: false, alt: "D")[$D$]) and 30 non-deaths (#math.equation(block: false, alt: "C")[$C$] or #math.equation(block: false, alt: "C L")[$C L$]) were corrected predicted. However, 16 predictions of death were ultimately non-deaths. This is a much more desirable error than the reverse (prediction of non-death when in fact the patient will die), so the model seems to be appropriate for the task of predicting cirrhosis fatalities.


  
  
    
=== Working with Big Data


    In the rapidly evolving landscape of machine learning, the concept of big data looms large. As we saw in Handling Large Datasets, large datasets, often termed #strong[big data], refers to extremely large and complex datasets that are beyond the capacity of traditional data processing and analysis tools to efficiently handle, manage, and analyze. Big data is characterized by the “three Vs”:


    - #strong[Volume:] Big data involves massive amounts of data, often ranging from terabytes (#math.equation(block: false, alt: "2 to the power 40 approximately equals 10 to the power 12")[$2^(40) ≈ 10^(12)$] bytes) to as large as petabytes (#math.equation(block: false, alt: "2 to the power 50 approximately equals 10 to the power 15")[$2^(50) ≈ 10^(15)$] bytes) and beyond. Where does such vast amounts of data originate? One example is #link("https://openstax.org/r/commoncrawl")[the Common Crawl dataset], which contains petabytes of data consisting of the raw web page data, metadata, and other information available from the internet. It is likely that there will be datasets containing exabytes (#math.equation(block: false, alt: "2 to the power 60 approximately equals 10 to the power 18")[$2^(60) ≈ 10^(18)$] bytes) of information in the near future.
- #strong[Velocity:] In some important real-world applications, data is generated and collected at an incredibly high speed. This continuous flow of data may be streamed in real time or stored in huge databases for later analysis. One example is in the area of very high-definition video, which can be captured, stored, and analyzed at a rate of gigabits per hour.
- #strong[Variety:] Big data also encompasses diverse types of data, including structured data (e.g., databases), unstructured data (e.g., text, images, videos), and anything in between. This variety adds considerable complexity to data management and analysis. Consider all the various sources of data and how different each may be in transmission and storage. From social media interactions and e-commerce transactions to sensor data from Internet of Things (IoT) devices, data streams in at a pace, scale, and variety never before seen. Moreover, few will be able to predict the brand-new ways that data will be used and gathered in the future!


    
      
==== Data Cleaning and Mining


      The most time-consuming aspect of data science has traditionally been #strong[data cleaning], a process we discussed in Data Cleaning and Preprocessing. Datasets are often messy, missing data, and likely to contain errors or typos. These issues are vastly compounded when considering big data. While it may take a person an hour or so to look through a dataset with a few hundred entries to spot and fix errors or deal with incomplete data, such work becomes impossible to do by hand when there are millions or billions of entries. The task of data cleaning at scale must be handled by technology; however, traditional data cleaning tools may be ill-equipped to handle big data because of the sheer volume. Fortunately, there are tools that can process large amounts of data in parallel.


    
    
      
==== Data Mining


      #strong[Data mining] is the process of discovering patterns, trends, and insights from large datasets. It involves using various techniques from machine learning, statistics, and database systems to uncover valuable knowledge hidden within data. Data mining aims to transform raw data into actionable information that can be used for decision-making, prediction, and problem-solving and is often the next step after data cleaning. Common data mining tasks include clustering, classification, regression, and anomaly detection. For unlabeled data, unsupervised machine learning algorithms may be used to discover clusters in smaller samples of the data, which can then be assigned labels that would be used for more far-ranging classification or regression analysis.


      #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
#emph[Tools for Data Mining]


        Tools for large-scale data mining include #link("https://openstax.org/r/spark")[Apache Spark], which uses a distributed file system (HDFS) to store and process data across clusters and also relies on in-memory processing, and #link("https://openstax.org/r/flink")[Apache Flink], which enables real-time data analytics. See this #link("https://openstax.org/r/macrometa")[Macrometa article] for a comparison of the two programs.
]


    
    
      
==== Methods for Working with Big Data


      Machine learning in the era of big data calls for scalability and adaptability. Traditional analysis methods, like linear regression, logistic regression, decision trees, and Bayesian classifiers, remain valuable tools. However, they often benefit from enhancements and new strategies tailored for big data environments:


      - #strong[Parallel processing:] Techniques that distribute computation across multiple nodes or cores, like parallelized decision trees and ensemble methods, can significantly speed up model training.
- #strong[Feature selection:] Not all features of a huge dataset may be used at once. Advanced feature selection algorithms are necessary to isolate only the most significant features useful for predictions.
- #strong[Progressive learning:] Some big data scenarios require models that can learn incrementally from data streams. Machine learning algorithms will need to adapt to changing data in real time.
- #strong[Deep learning:] Deep neural networks, with their capacity to process vast volumes of data and learn intricate patterns, have become increasingly important in big data applications. We will delve into this topic in Deep Learning and AI Basics.
- #strong[Dimensionality reduction:] Techniques like principal component analysis (PCA) help reduce the dimensionality of data, making it more manageable while retaining essential information. (Note: Dimension reduction falls outside the scope of this text.)


      When the volume of data is large, the simplest and most efficient algorithms would typically be chosen, in which speed is more important than accuracy. For example, spam filtering for email often utilizes multinomial Bayes’ classification. In the age of instant digital communication, your email inbox can quickly become flooded with hundreds of meaningless or even malicious messages (i.e., spam) in a day, and so it is very important to have a method of classifying and removing the spam. Algorithms easily scale to work on big data, while other methods such as decision trees and random forests require significant overhead and would not scale up as easily.


      #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
Datasets


        #emph[Note: The primary datasets referenced in the chapter code may also be #link("https://openstax.org/r/drive6")[downloaded here]].
]


    
  
  
    
      
==== Project A: Comparing k-Means and DBScan


      In this chapter, there were two datasets used to illustrate k-means clustering and DBScan, #link("https://openstax.org/r/FungusLocations")[#strong[FungusLocations.csv]] and #link("https://openstax.org/r/DBScanExample")[#strong[DBScanExample.csv]]. Run each algorithm on the other dataset. In other words, use DBScan to classify the points of #link("https://openstax.org/r/FungusLocations")[FungusLocations.csv] and k-means to cluster the points of #link("https://openstax.org/r/DBScanExample")[DBScanExample.csv]. Discuss the results, comparing the performance of both algorithms on each dataset. If there were any issues with the classifications, discuss possible reasons why those issues came up.


    
    
      
==== Project B: Building a Decision Tree to Predict College Completion


      Build a decision tree classifier based on the dataset #link("https://openstax.org/r/CollegeCompletionData")[#strong[CollegeCompletionData.csv]] to classify students as likely to complete college (or not) based on their GPA and in-state status. Experiment with different ratios of training to testing data and different pruning techniques to find a model with the best accuracy. Generate some data points with random GPAs and in-state statuses and use your model to predict college completion on your new data.


    
    
      
==== Project C: Predicting Outcomes in Liver Disease Patients


      Analyze the dataset #link("https://openstax.org/r/cirrhosis")[#strong[cirrhosis.csv]] to predict labels #math.equation(block: false, alt: "D")[$D$], #math.equation(block: false, alt: "C")[$C$], and #math.equation(block: false, alt: "C L")[$C L$] based on various combinations of the feature columns, using (a) random forest and (b) multiple logistic regression. Compare the accuracy of your models with the Gaussian naïve Bayes classifier that was produced in Gaussian Naive Bayes for Continuous Probabilities.


    
  
  
  
  
  
    1. Emspak, J. (2016, December 29). How a machine learns prejudice.#emph[Scientific American]. https://www.scientificamerican.com/article/how-a-machine-learns-prejudice/