Add HW0

.gitignore

@@ -0,0 +1,9 @@
+# LaTeX compile artifacts
+*.cls
+*.aux
+*.fdb_latexmk
+*.fls
+*.log
+*.out
+*.synctex.gz
+_minted-hw0_solutions/

HW0/common.sty

@@ -0,0 +1,126 @@
+\usepackage{amsmath}
+\usepackage{amssymb}
+\usepackage{fullpage,graphicx}
+\usepackage{tikz}
+\usetikzlibrary{patterns}
+\pagestyle{empty}
+\usepackage{subfig}
+\usepackage{comment}
+
+\newcommand{\boldA}{\mathbf{A}}
+\newcommand{\boldB}{\mathbf{B}}
+\newcommand{\boldC}{\mathbf{C}}
+\newcommand{\boldD}{\mathbf{D}}
+\newcommand{\boldE}{\mathbf{E}}
+\newcommand{\boldF}{\mathbf{F}}
+\newcommand{\boldG}{\mathbf{G}}
+\newcommand{\boldH}{\mathbf{H}}
+\newcommand{\boldI}{\mathbf{I}}
+\newcommand{\boldJ}{\mathbf{J}}
+\newcommand{\boldK}{\mathbf{K}}
+\newcommand{\boldL}{\mathbf{L}}
+\newcommand{\boldM}{\mathbf{M}}
+\newcommand{\boldN}{\mathbf{N}}
+\newcommand{\boldO}{\mathbf{O}}
+\newcommand{\boldP}{\mathbf{P}}
+\newcommand{\boldQ}{\mathbf{Q}}
+\newcommand{\boldR}{\mathbf{R}}
+\newcommand{\boldS}{\mathbf{S}}
+\newcommand{\boldT}{\mathbf{T}}
+\newcommand{\boldU}{\mathbf{U}}
+\newcommand{\boldV}{\mathbf{V}}
+\newcommand{\boldW}{\mathbf{W}}
+\newcommand{\boldX}{\mathbf{X}}
+\newcommand{\boldY}{\mathbf{Y}}
+\newcommand{\boldZ}{\mathbf{Z}}
+\newcommand{\bolda}{\mathbf{a}}
+\newcommand{\boldb}{\mathbf{b}}
+\newcommand{\boldc}{\mathbf{c}}
+\newcommand{\boldd}{\mathbf{d}}
+\newcommand{\bolde}{\mathbf{e}}
+\newcommand{\boldf}{\mathbf{f}}
+\newcommand{\boldg}{\mathbf{g}}
+\newcommand{\boldh}{\mathbf{h}}
+\newcommand{\boldi}{\mathbf{i}}
+\newcommand{\boldj}{\mathbf{j}}
+\newcommand{\boldk}{\mathbf{k}}
+\newcommand{\boldl}{\mathbf{l}}
+\newcommand{\boldm}{\mathbf{m}}
+\newcommand{\boldn}{\mathbf{n}}
+\newcommand{\boldo}{\mathbf{o}}
+\newcommand{\boldp}{\mathbf{p}}
+\newcommand{\boldq}{\mathbf{q}}
+\newcommand{\boldr}{\mathbf{r}}
+\newcommand{\bolds}{\mathbf{s}}
+\newcommand{\boldt}{\mathbf{t}}
+\newcommand{\boldu}{\mathbf{u}}
+\newcommand{\boldv}{\mathbf{v}}
+\newcommand{\boldw}{\mathbf{w}}
+\newcommand{\boldx}{\mathbf{x}}
+\newcommand{\boldy}{\mathbf{y}}
+\newcommand{\boldz}{\mathbf{z}}
+
+\newcommand{\mcA}{\mathcal{A}}
+\newcommand{\mcB}{\mathcal{B}}
+\newcommand{\mcC}{\mathcal{C}}
+\newcommand{\mcD}{\mathcal{D}}
+\newcommand{\mcE}{\mathcal{E}}
+\newcommand{\mcF}{\mathcal{F}}
+\newcommand{\mcG}{\mathcal{G}}
+\newcommand{\mcH}{\mathcal{H}}
+\newcommand{\mcI}{\mathcal{I}}
+\newcommand{\mcJ}{\mathcal{J}}
+\newcommand{\mcK}{\mathcal{K}}
+\newcommand{\mcL}{\mathcal{L}}
+\newcommand{\mcM}{\mathcal{M}}
+\newcommand{\mcN}{\mathcal{N}}
+\newcommand{\mcO}{\mathcal{O}}
+\newcommand{\mcP}{\mathcal{P}}
+\newcommand{\mcQ}{\mathcal{Q}}
+\newcommand{\mcR}{\mathcal{R}}
+\newcommand{\mcS}{\mathcal{S}}
+\newcommand{\mcT}{\mathcal{T}}
+\newcommand{\mcU}{\mathcal{U}}
+\newcommand{\mcV}{\mathcal{V}}
+\newcommand{\mcW}{\mathcal{W}}
+\newcommand{\mcX}{\mathcal{X}}
+\newcommand{\mcY}{\mathcal{Y}}
+\newcommand{\mcZ}{\mathcal{Z}}
+
+\newcommand{\reals}{\ensuremath{\mathbb{R}}}
+\newcommand{\integers}{\ensuremath{\mathbb{Z}}}
+\newcommand{\rationals}{\ensuremath{\mathbb{Q}}}
+\newcommand{\naturals}{\ensuremath{\mathbb{N}}}
+\newcommand{\trans}{\ensuremath{\mathsf{T}}}
+\newcommand{\ident}{\mathbf{I}}
+\newcommand{\bzero}{\mathbf{0}}
+
+\newcommand{\balpha}{\mathbf{\alpha}}
+\newcommand{\bbeta}{\mathbf{\beta}}
+\newcommand{\bdelta}{\mathbf{\delta}}
+\newcommand{\boldeta}{\mathbf{\eta}}
+\newcommand{\bkappa}{\mathbf{\kappa}}
+\newcommand{\bgamma}{\mathbf{\gamma}}
+\newcommand{\bmu}{\boldsymbol{\mu}}
+\newcommand{\bphi}{\mathbf{\phi}}
+\newcommand{\bpi}{\boldsymbol{\pi}}
+\newcommand{\bpsi}{\mathbf{\psi}}
+\newcommand{\bsigma}{\mathbf{\sigma}}
+\newcommand{\btheta}{\mathbf{\theta}}
+\newcommand{\bxi}{\mathbf{\xi}}
+\newcommand{\bGamma}{\mathbf{\Gamma}}
+\newcommand{\bLambda}{\mathbf{\Lambda}}
+\newcommand{\bOmega}{\mathbf{\Omega}}
+\newcommand{\bPhi}{\mathbf{\Phi}}
+\newcommand{\bPi}{\mathbf{\Pi}}
+\newcommand{\bPsi}{\mathbf{\Psi}}
+\newcommand{\bSigma}{\mathbf{\Sigma}}
+\newcommand{\bTheta}{\mathbf{\Theta}}
+\newcommand{\bUpsilon}{\mathbf{\Upsilon}}
+\newcommand{\bXi}{\mathbf{\Xi}}
+\newcommand{\bepsilon}{\mathbf{\epsilon}}
+
+\def\argmin{\operatornamewithlimits{arg\,min}}
+
+\newcommand{\given}{\,|\,}
+\newcommand{\distNorm}{\mathcal{N}}

HW0/hw0.tex

@@ -0,0 +1,149 @@
+\documentclass{harvardml}
+
+% Authors: Amir Shanehsazzadeh, Andrew Kim, Nari Johnson (Jan 2021)
+% Edited by: Max Guo, Raphael Pellegrin, Katherine Tian (Jan 2022)
+% Edited once more by: William Tong, Matthew Nazari, Skyler Wu (Jan 2023)
+
+% Adapted from CS281 Fall 2019 section 0 notes
+
+% This tex file relies on
+% the presence of two files:
+% harvardml.cls and common.sty
+
+\course{CS181-s23}
+\assignment{CS181 HW0}
+\duedate{January 26, 2023 at 11:59 PM}
+
+\usepackage{comment}
+\usepackage{url}
+\usepackage{amsfonts, amsmath, amsthm}
+\usepackage{listings}
+\usepackage[shortlabels]{enumitem}
+\usepackage{hyperref}
+
+\theoremstyle{definition}
+\newtheorem{defn}{Definition}[section]
+\theoremstyle{plain}
+\usepackage[textsize=tiny]{todonotes}
+
+% Some useful macros.
+\newcommand{\given}{\,|\,}
+\newcommand{\R}{\mathbb{R}}
+\newcommand{\C}{\mathbb{C}}
+\newcommand{\E}{\mathbb{E}}
+\newcommand{\var}{\text{Var}}
+\newcommand{\cov}{\text{Cov}}
+\newcommand{\p}{\partial}
+\newcommand{\mba}{\mathbf{a}}
+\newcommand{\mbb}{\mathbf{b}}
+\newcommand{\mbx}{\mathbf{x}}
+\newcommand{\mcX}{\mathcal{X}}
+\newcommand{\mcY}{\mathcal{Y}}
+\newcommand{\boldw}{\mathbf{w}}
+\newcommand{\mbxt}{\tilde{\mathbf{x}}}
+\newcommand{\Sigmat}{\tilde{\Sigma}}
+\newcommand{\mbz}{\mathbf{z}}
+\newcommand{\mbw}{\mathbf{w}}
+\newcommand{\mcN}{\mathcal{N}}
+\newcommand{\mcP}{\mathcal{P}}
+\newcommand{\eps}{\epsilon}
+\newcommand{\trans}{\intercal}
+\newcommand{\Ut}{\tilde{U}}
+\DeclareMathOperator*{\argmax}{arg\,max}
+\newcommand{\angstrom}{\textup{\AA}}
+\renewcommand{\v}[1]{\mathbf{#1}}
+
+
+\usepackage{xcolor}
+\newcount\Comments  % 0 suppresses notes to selves in text
+\Comments = 1
+\newcommand{\kibitz}[2]{\ifnum\Comments=1{\color{#1}{#2}}\fi}
+\newcommand{\dcp}[1]{\kibitz{blue}{[DCP: #1]}}
+
+\begin{document}
+
+\noindent Welcome to CS181! The purpose of this assignment is to help assess your readiness for this course. \textbf{This assignment will be graded for completion and effort.} If you encounter any difficulty with these problems, fear not! We will have sections in the first week of class reviewing the math, statistics, and coding pre-requisites for this course. TFs will also directly discuss relevant problems from this HW0 in these sections. You are, of course, more than welcome to swing by office hours and post questions on Ed.
+
+\begin{enumerate}
+    \item Please type your solutions after the corresponding problems using this \LaTeX\ template, and start each problem on a new page.
+    \item Please submit the \textbf{writeup PDF to the Gradescope assignment `HW0'}. Remember to assign pages for each question.
+    \item Please submit your \textbf{\LaTeX\ file and code files (i.e., anything ending in \texttt{.py}, \texttt{.ipynb}, or \texttt{.tex}) to the Gradescope assignment `HW0 - Supplemental'}. 
+    \item You may use a \textbf{maximum of 2 late days} on this assignment.  Late days will be counted based on the latest of your submissions. 
+\end{enumerate}
+
+\begin{problem}[Modeling Linear Trends - Linear Algebra Review]
+In this class we will be exploring the question of ``how do we model the trend in a dataset" under different guises. In this problem, we will explore the algebra of modeling a linear trend in data. We call the process of finding a model that capture the trend in the data, ``fitting the model."\\
+
+\noindent \textbf{Learning Goals:} In this problem, you will practice translating machine learning goals (``modeling trends in data") into mathematical formalism using linear algebra. You will explore how the right mathematical formalization can help us express our modeling ideas unambiguously and provide ways for us to analyze different pathways to meeting our machine learning goals.\\
+
+\noindent Let's consider a dataset consisting of two points $\mathcal{D} = \{(x_1, y_1), (x_2, y_2)\}$, where $x_n, y_n$ are scalars for $n=1, 2$. Recall that the equation of a line in 2-dimensions can be written: $y = w_0 + w_1x$. 
+\begin{enumerate}
+    \item Write a system of linear equations determining the coefficients $w_0, w_1$ of the line passing through the points in our dataset $\mathcal{D}$ and analytically solve for $w_0, w_1$ by solving this system of linear equations (i.e., using substitution). Please show your work.
+    \item Write the above system of linear equations in matrix notation, so that you have a matrix equation of the form $\mathbf{y} = \mathbf{X}\mathbf{w}$, where $\mathbf{y}, \mathbf{w} \in \mathbb{R}^2$ and $\mathbf{X} \in \mathbb{R}^{2\times 2}$. It suffices to write out what $\mathbf{X}$, $\mathbf{y}$, and $\mathbf{w}$ should look like in terms of $x_1$, $x_2$, $y_1$, $y_2$, $w_0$, $w_1$, and any other necessary constants. Please show your reasoning and supporting intermediate steps.
+    \item Using properties of matrices, characterize exactly when an unique solution for  $\mathbf{w}=\left(w_0 \; w_1 \right)^{T}$ exists. In other words, what must be true about your dataset in order for there to be a unique solution for $\mathbf{w}$? When the solution for $\mathbf{w}$ exists (and is unique), write out, as a matrix expression, its analytical form (i.e., write $\mathbf{w}$ in terms of $\mathbf{X}$ and $\mathbf{y}$).
+
+    Hint: What special property must our $\mathbf{X}$ matrix possess? What must be true about our data points in $\mathcal{D}$ for this special property to hold?
+    \item Compute $\mathbf{w}$ by hand via your matrix expression in (3) and compare it with your solution in (1). Do your final answers match? What is one advantage for phrasing the problem of fitting the model in terms of matrix notation? 
+    \item In real-life, we often work with datasets that consist of hundreds, if not millions, of points. In such cases, does our analytical expression for $\mathbf{w}$ that we derived in (3) apply immediately to the case when $\mathcal{D}$ consists of more than two points? Why or why not?
+\end{enumerate}
+
+\end{problem}
+
+\begin{problem}[Optimizing Objectives - Calculus Review]
+In this class, we will write real-life goals we want our model to achieve into a mathematical expression and then find the optimal settings of the model that achieves these goals. The formal framework we will employ is that of mathematical optimization. Although the mathematics of optimization can be quite complex and deep, we have all encountered basic optimization problems in our first calculus class!\\
+
+\noindent \textbf{Learning Goals:} In this problem, we will explore how to formalize real-life goals as mathematical optimization problems. We will also investigate under what conditions these optimization problems have solutions.\\
+
+\noindent In her most recent work-from-home shopping spree, Nari decided to buy several house plants. \textit{Her goal is to make them to grow as tall as possible.} After perusing the internet, Nari learns that the height $y$ in mm of her Weeping Fig plant can be directly modeled as a function of the oz of water $x$ she gives it each week:
+$$y = - 3x^2 + 72x + 70.$$
+\begin{enumerate}
+    \item Based on the above formula, is Nari's goal achievable: does the plant have a maximum height? Why or why not? Does her goal have a unique solution - i.e. is there one special watering schedule that would acheive the maximum height (if it exists)?
+
+    Hint: plot this function. In your solution, words like ``convex" and ``concave" may be helpful.
+    \item Using calculus, find how many oz per week should Nari water her plant in order to maximize its height. With this much water, how tall will her plant grow?
+
+    Hint: solve analytically for the critical points of the height function (i.e., where the derivative of the function is zero).  For each critical point, use the second-derivative test to identify if each point is a  max or min point, and use arguments about the global structure (e.g., concavity or convexity) of the function to argue whether this is a local or global optimum.
+\end{enumerate}
+Now suppose that Nari want to optimize both the amount of water $x_1$ (in oz) *and* the amount of direct sunlight $x_2$ (in hours) to provide for her plants. After extensive research, she decided that the height $y$ (in mm) of her plants can be modeled as a two variable function:
+
+%$$y = f(x_1, x_2) = -e^{1 - (x_1 + x_2^2)} + 400.$$
+$$y = f(x_1, x_2) = \exp\left(-(x_1 - 2)^2 - (x_2 - 1)^2 \right)$$
+\begin{enumerate}
+    \setcounter{enumi}{2}
+    \item Using \texttt{matplotlib}, visualize in 3D the height function as a function of $x_1$ and $x_2$ using the \texttt{plot\_surface} utility for $(x_1, x_2) \in (0, 6) \times (0, 6)$. Use this visualization to argue why there exists a unique solution to Nari's optimization problem on the specified intervals for $x_1$ and $x_2$.
+
+    Remark: in this class, we will learn about under what conditions do \textit{multivariate} optimization problems have unique global optima (and no, the second derivative test doesn't exactly generalize directly). Looking at the visualization you produced and the expression for $f(x_1, x_2)$, do you have any ideas for why this problem is guaranteed to have a global maxima? You do not need to write anything responding to this -- this is simply food for thought and a preview for the semester.
+\end{enumerate}
+
+\end{problem}
+
+\begin{problem}[Reasoning about Randomness - Probability and Statistics Review]
+In this class, one of our main focuses is to model the unexpected variations in real-life phenomena using the formalism of random variables. In this problem, we will use random variables to model how much time it takes an USPS package processing system to process packages that arrive in a day.\\
+
+\noindent \textbf{Learning Goals:} In this problem, you will analyze random variables and their distributions both analytically and computationally. You will also practice drawing connections between said analytical and computational conclusions.\\
+
+\noindent Consider the following model for packages arriving at the US Postal Service (USPS):
+\begin{itemize}
+    \item Packages arrive randomly in any given hour according to a Poisson distribution. That is, the number of packages in a given hour $N$ is distributed $Pois(\lambda)$, with $\lambda = 3$.
+    \item Each package has a random size $S$ (measured in $in^3$) and weight $W$ (measured in pounds), with joint distribution
+    $$(S, W)^{T} \sim \mathcal{N}\left( \boldsymbol{\mu}, \boldsymbol{\Sigma}\right) \text{, with } \boldsymbol{\mu} = \begin{bmatrix} 120 \\ 4 \end{bmatrix} \text{ and } \boldsymbol{\Sigma} = \begin{bmatrix} 1.5 & 1 \\ 1 & 1.5 \end{bmatrix}.$$
+    \item Processing time $T$ (in seconds) for each package is given by $T = 60 + 0.6 W + 0.2 S + \epsilon$, where $\epsilon$ is a random noise variable with Gaussian distribution $\epsilon \sim \mathcal{N}(0, 5)$.
+\end{itemize}
+For this problem, you may find the \texttt{multivariate\_normal} module from \texttt{scipy.stats} especially helpful. You may also find the \texttt{seaborn.histplot} function quite helpful. 
+\begin{enumerate}
+    \item Perform the following tasks:
+    \begin{enumerate}
+        \item Visualize the Bivariate Gaussian distribution for the size $S$ and weight $W$ of the packages by sampling 500 times from the joint distribution of $S$ and $W$ and generating a bivariate histogram of your $S$ and $W$ samples.
+        \item Empirically estimate the most likely combination of size and weight of a package by finding the bin of your bivariate histogram (i.e., specify both a value of $S$ and a value of $W$) with the highest frequency. A visual inspection is sufficient -- you do not need to be incredibly precise.  How close are these empirical values to the theoretical expected size and expected weight of a package, according to the given Bivariate Gaussian distribution?
+    \end{enumerate}
+    \item For 1001 evenly-spaced values of $W$ between $0$ and $10$, plot $W$ versus the joint Bivariate Gaussian PDF $p(W, S)$ with $S$ fixed at $S=118$. Repeat this procedure for $S$ fixed at $S=122$. Comparing these two PDF plots, what can you say about the correlation of random variables $S$ and $W$?
+    \item Give one reason for why the Gaussian distribution is an appropriate model for the size and weight of packages. Give one reason for why it may not be appropriate.
+    \item Because $T$ is a linear combination of random variables, it itself is a random variable. Using properties of expectations and variance, please compute $\mathbb{E}(T)$ and $\mathrm{Var}(T)$ analytically.
+    \item Let us treat the \textit{total} amount of time it takes to process \textit{all} packages received at the USPS office within \textit{an entire day} (assuming a single day is $24$ hours long) as a random variable $T^{*}$. 
+    \begin{enumerate}
+        \item Write a function to simulate draws from the distribution of $T^{*}$. 
+        \item Using your function, empirically estimate the mean and standard deviation of $T^{*}$ by generating $1000$ samples from the distribution of $T^{*}$.
+    \end{enumerate}
+\end{enumerate}
+\end{problem}
+\end{document}