proj5 initialized

proj5/machinelearning/autograder.py

@@ -0,0 +1,637 @@
+# A custom autograder for this project
+
+################################################################################
+# A mini-framework for autograding
+################################################################################
+
+import optparse
+import sys
+import traceback
+
+
+class WritableNull:
+    def write(self, string):
+        pass
+
+    def flush(self):
+        pass
+
+class Tracker(object):
+    def __init__(self, questions, maxes, prereqs, mute_output):
+        self.questions = questions
+        self.maxes = maxes
+        self.prereqs = prereqs
+
+        self.points = {q: 0 for q in self.questions}
+
+        self.current_question = None
+
+        self.current_test = None
+        self.points_at_test_start = None
+        self.possible_points_remaining = None
+
+        self.mute_output = mute_output
+        self.original_stdout = None
+        self.muted = False
+
+    def mute(self):
+        if self.muted:
+            return
+
+        self.muted = True
+        self.original_stdout = sys.stdout
+        sys.stdout = WritableNull()
+
+    def unmute(self):
+        if not self.muted:
+            return
+
+        self.muted = False
+        sys.stdout = self.original_stdout
+
+    def begin_q(self, q):
+        assert q in self.questions
+        text = 'Question {}'.format(q)
+        print('\n' + text)
+        print('=' * len(text))
+
+        for prereq in sorted(self.prereqs[q]):
+            if self.points[prereq] < self.maxes[prereq]:
+                print("""*** NOTE: Make sure to complete Question {} before working on Question {},
+*** because Question {} builds upon your answer for Question {}.
+""".format(prereq, q, q, prereq))
+                return False
+
+        self.current_question = q
+        self.possible_points_remaining = self.maxes[q]
+        return True
+
+    def begin_test(self, test_name):
+        self.current_test = test_name
+        self.points_at_test_start = self.points[self.current_question]
+        print("*** {}) {}".format(self.current_question, self.current_test))
+        if self.mute_output:
+            self.mute()
+
+    def end_test(self, pts):
+        if self.mute_output:
+            self.unmute()
+        self.possible_points_remaining -= pts
+        if self.points[self.current_question] == self.points_at_test_start + pts:
+            print("*** PASS: {}".format(self.current_test))
+        elif self.points[self.current_question] == self.points_at_test_start:
+            print("*** FAIL")
+
+        self.current_test = None
+        self.points_at_test_start = None
+
+    def end_q(self):
+        assert self.current_question is not None
+        assert self.possible_points_remaining == 0
+        print('\n### Question {}: {}/{} ###'.format(
+            self.current_question,
+            self.points[self.current_question],
+            self.maxes[self.current_question]))
+
+        self.current_question = None
+        self.possible_points_remaining = None
+
+    def finalize(self):
+        import time
+        print('\nFinished at %d:%02d:%02d' % time.localtime()[3:6])
+        print("\nProvisional grades\n==================")
+
+        for q in self.questions:
+          print('Question %s: %d/%d' % (q, self.points[q], self.maxes[q]))
+        print('------------------')
+        print('Total: %d/%d' % (sum(self.points.values()),
+            sum([self.maxes[q] for q in self.questions])))
+
+        print("""
+Your grades are NOT yet registered.  To register your grades, make sure
+to follow your instructor's guidelines to receive credit on your project.
+""")
+
+    def add_points(self, pts):
+        self.points[self.current_question] += pts
+
+TESTS = []
+PREREQS = {}
+def add_prereq(q, pre):
+    if isinstance(pre, str):
+        pre = [pre]
+
+    if q not in PREREQS:
+        PREREQS[q] = set()
+    PREREQS[q] |= set(pre)
+
+def test(q, points):
+    def deco(fn):
+        TESTS.append((q, points, fn))
+        return fn
+    return deco
+
+def parse_options(argv):
+    parser = optparse.OptionParser(description = 'Run public tests on student code')
+    parser.set_defaults(
+        edx_output=False,
+        gs_output=False,
+        no_graphics=False,
+        mute_output=False,
+        check_dependencies=False,
+        )
+    parser.add_option('--edx-output',
+                        dest = 'edx_output',
+                        action = 'store_true',
+                        help = 'Ignored, present for compatibility only')
+    parser.add_option('--gradescope-output',
+                        dest = 'gs_output',
+                        action = 'store_true',
+                        help = 'Ignored, present for compatibility only')
+    parser.add_option('--question', '-q',
+                        dest = 'grade_question',
+                        default = None,
+                        help = 'Grade only one question (e.g. `-q q1`)')
+    parser.add_option('--no-graphics',
+                        dest = 'no_graphics',
+                        action = 'store_true',
+                        help = 'Do not display graphics (visualizing your implementation is highly recommended for debugging).')
+    parser.add_option('--mute',
+                        dest = 'mute_output',
+                        action = 'store_true',
+                        help = 'Mute output from executing tests')
+    parser.add_option('--check-dependencies',
+                        dest = 'check_dependencies',
+                        action = 'store_true',
+                        help = 'check that numpy and matplotlib are installed')
+    (options, args) = parser.parse_args(argv)
+    return options
+
+def main():
+    options = parse_options(sys.argv)
+    if options.check_dependencies:
+        check_dependencies()
+        return
+
+    if options.no_graphics:
+        disable_graphics()
+
+    questions = set()
+    maxes = {}
+    for q, points, fn in TESTS:
+        questions.add(q)
+        maxes[q] = maxes.get(q, 0) + points
+        if q not in PREREQS:
+            PREREQS[q] = set()
+
+    questions = list(sorted(questions))
+    if options.grade_question:
+        if options.grade_question not in questions:
+            print("ERROR: question {} does not exist".format(options.grade_question))
+            sys.exit(1)
+        else:
+            questions = [options.grade_question]
+            PREREQS[options.grade_question] = set()
+
+    tracker = Tracker(questions, maxes, PREREQS, options.mute_output)
+    for q in questions:
+        started = tracker.begin_q(q)
+        if not started:
+            continue
+
+        for testq, points, fn in TESTS:
+            if testq != q:
+                continue
+            tracker.begin_test(fn.__name__)
+            try:
+                fn(tracker)
+            except KeyboardInterrupt:
+                tracker.unmute()
+                print("\n\nCaught KeyboardInterrupt: aborting autograder")
+                tracker.finalize()
+                print("\n[autograder was interrupted before finishing]")
+                sys.exit(1)
+            except:
+                tracker.unmute()
+                print(traceback.format_exc())
+            tracker.end_test(points)
+        tracker.end_q()
+    tracker.finalize()
+
+################################################################################
+# Tests begin here
+################################################################################
+
+import numpy as np
+import matplotlib
+import contextlib
+
+from torch import nn, Tensor
+import torch
+import backend
+
+def check_dependencies():
+    import matplotlib.pyplot as plt
+    fig, ax = plt.subplots(1, 1)
+    ax.set_xlim([-1, 1])
+    ax.set_ylim([-1, 1])
+    line, = ax.plot([], [], color="black")
+    plt.show(block=False)
+
+    for t in range(400):
+        angle = t * 0.05
+        x = np.sin(angle)
+        y = np.cos(angle)
+        line.set_data([x,-x], [y,-y])
+        fig.canvas.draw_idle()
+        fig.canvas.start_event_loop(1e-3)
+
+def disable_graphics():
+    backend.use_graphics = False
+
+@contextlib.contextmanager
+def no_graphics():
+    old_use_graphics = backend.use_graphics
+    backend.use_graphics = False
+    yield
+    backend.use_graphics = old_use_graphics
+
+def verify_node(node, expected_type, expected_shape, method_name):
+    if expected_type == 'parameter':
+        assert node is not None, (
+            "{} should return an instance of nn.Parameter, not None".format(method_name))
+        assert isinstance(node, nn.Parameter), (
+            "{} should return an instance of nn.Parameter, instead got type {!r}".format(
+            method_name, type(node).__name__))
+    elif expected_type == 'loss':
+        assert node is not None, (
+            "{} should return an instance a loss node, not None".format(method_name))
+        assert isinstance(node, (nn.modules.loss._Loss)), (
+            "{} should return a loss node, instead got type {!r}".format(
+            method_name, type(node).__name__))
+    elif expected_type == 'tensor':
+        assert node is not None, (
+            "{} should return a node object, not None".format(method_name))
+        assert isinstance(node, Tensor), (
+            "{} should return a node object, instead got type {!r}".format(
+            method_name, type(node).__name__))
+    else:
+        assert False, "If you see this message, please report a bug in the autograder"
+
+    if expected_type != 'loss':
+        assert all([(expected is '?' or actual == expected) for (actual, expected) in zip(node.detach().numpy().shape, expected_shape)]), (
+            "{} should return an object with shape {}, got {}".format(
+                method_name, expected_shape, node.shape))
+
+@test('q1', points=6)
+def check_perceptron(tracker):
+    import models
+
+    print("Sanity checking perceptron...")
+    np_random = np.random.RandomState(0)
+    
+    # Check that the perceptron weights are initialized to a single vector with `dimensions` entries.
+    for dimensions in range(1, 10):
+        p = models.PerceptronModel(dimensions)
+        p_weights = p.get_weights()
+
+        number_of_parameters = 0
+
+        for param in p.parameters():
+            number_of_parameters += 1
+            verify_node(param, 'parameter', (1, dimensions), 'PerceptronModel.parameters()')
+
+        assert number_of_parameters == 1, ('Perceptron Model should only have 1 parameter')
+
+    # Check that run returns a Tensor, and that the score in the node is correct
+    for dimensions in range(1, 10):
+        p = models.PerceptronModel(dimensions)
+        point = np_random.uniform(-10, 10, (1, dimensions))
+        score = p.run(Tensor(point))
+        verify_node(score, 'tensor', (1,), "PerceptronModel.run()")
+        calculated_score = score.item()
+        
+        # Compare run output to actual value
+        for param in p.parameters():
+            expected_score = float(np.dot(point.flatten(), param.detach().numpy().flatten()))
+
+        assert np.isclose(calculated_score, expected_score), (
+            "The score computed by PerceptronModel.run() ({:.4f}) does not match the expected score ({:.4f})".format(
+            calculated_score, expected_score))
+
+    # Check that get_prediction returns the correct values, including the
+    # case when a point lies exactly on the decision boundary
+    for dimensions in range(1, 10):
+        p = models.PerceptronModel(dimensions)
+        random_point = np_random.uniform(-10, 10, (1, dimensions))
+        for point in (random_point, np.zeros_like(random_point)):
+            prediction = p.get_prediction(Tensor(point))
+            assert prediction == 1 or prediction == -1, (
+                "PerceptronModel.get_prediction() should return 1 or -1, not {}".format(
+                prediction))
+
+            expected_prediction = np.where(np.dot(point, p.get_weights().data.T) >= 0, 1, -1).item()
+            assert prediction == expected_prediction, (
+                "PerceptronModel.get_prediction() returned {}; expected {}".format(
+                    prediction, expected_prediction))
+
+    tracker.add_points(2) # Partial credit for passing sanity checks
+
+    print("Sanity checking perceptron weight updates...")
+
+    # Test weight updates. This involves constructing a dataset that
+    # requires 0 or 1 updates before convergence, and testing that weight
+    # values change as expected. Note that (multiplier < -1 or multiplier > 1)
+    # must be true for the testing code to be correct.
+    dimensions = 2
+    for multiplier in (-5, -2, 2, 5):
+        p = models.PerceptronModel(dimensions)
+        orig_weights = p.get_weights().data.reshape((1, dimensions)).detach().numpy().copy()
+        if np.abs(orig_weights).sum() == 0.0:
+            # This autograder test doesn't work when weights are exactly zero
+            continue
+        
+        point = multiplier * orig_weights
+
+        sanity_dataset = backend.Custom_Dataset(
+            x=np.tile(point, (500, 1)),
+            y=np.ones((500, 1)) * -1.0
+        )
+        
+        p.train(sanity_dataset)
+        new_weights = p.get_weights().data.reshape((1, dimensions)).detach().numpy()
+
+        if multiplier < 0:
+            expected_weights = orig_weights
+        else:
+            expected_weights = orig_weights - point
+
+        if not np.all(new_weights == expected_weights):
+            print()
+            print("Initial perceptron weights were: [{:.4f}, {:.4f}]".format(
+                orig_weights[0,0], orig_weights[0,1]))
+            print("All data points in the dataset were identical and had:")
+            print("    x = [{:.4f}, {:.4f}]".format(
+                point[0,0], point[0,1]))
+            print("    y = -1")
+            print("Your trained weights were: [{:.4f}, {:.4f}]".format(
+                new_weights[0,0], new_weights[0,1]))
+            print("Expected weights after training: [{:.4f}, {:.4f}]".format(
+                expected_weights[0,0], expected_weights[0,1]))
+            print()
+            assert False, "Weight update sanity check failed"
+
+    print("Sanity checking complete. Now training perceptron")
+    model = models.PerceptronModel(3)
+    dataset = backend.PerceptronDataset(model)
+
+    model.train(dataset)
+    backend.maybe_sleep_and_close(1)
+
+    assert dataset.epoch != 0, "Perceptron code never iterated over the training data"
+
+    accuracy = np.mean(np.where(np.dot(dataset.x, model.get_weights().data.T) >= 0.0, 1.0, -1.0) == dataset.y)
+    if accuracy < 1.0:
+        print("The weights learned by your perceptron correctly classified {:.2%} of training examples".format(accuracy))
+        print("To receive full points for this question, your perceptron must converge to 100% accuracy")
+        return
+
+    tracker.add_points(4)
+
+@test('q2', points=6)
+def check_regression(tracker):
+    import models
+    model = models.RegressionModel()
+    dataset = backend.RegressionDataset(model=model)
+    detected_parameters = None
+
+    for batch_size in (1, 2, 4):
+        inp_x = torch.tensor(dataset.x[:batch_size], dtype=torch.float, requires_grad=True)
+        inp_y = torch.tensor(dataset.y[:batch_size], dtype=torch.float, requires_grad=True) 
+
+        loss =  model.get_loss(inp_x, inp_y)
+
+        verify_node(loss, 'tensor', (1,), "RegressionModel.get_loss()")
+    
+
+        grad_y = torch.autograd.grad(loss, inp_x, allow_unused=True, retain_graph=True)
+        grad_x = torch.autograd.grad(loss, inp_y, allow_unused=True, retain_graph=True)
+        
+        assert grad_x[0] != None, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)"
+        assert grad_y[0] != None, "Node returned from RegressionModel.get_loss() does not depend on the provided labels (y)"
+
+
+
+    tracker.add_points(2) # Partial credit for passing sanity checks
+
+    model.train(dataset)
+    backend.maybe_sleep_and_close(1)
+
+    data_x = torch.tensor(dataset.x,dtype=torch.float32)
+    labels = torch.tensor(dataset.y, dtype=torch.float32)
+    train_loss = model.get_loss(data_x, labels)
+    verify_node(train_loss, 'tensor', (1,), "RegressionModel.get_loss()")
+    train_loss = train_loss.item()
+
+    # Re-compute the loss ourselves: otherwise get_loss() could be hard-coded
+    # to always return zero
+    train_predicted = model(data_x)
+
+    verify_node(train_predicted, 'tensor', (dataset.x.shape[0], 1), "RegressionModel()")
+    error = labels - train_predicted
+    sanity_loss = torch.mean((error.detach())**2)
+
+    assert np.isclose(train_loss, sanity_loss), (
+        "RegressionModel.get_loss() returned a loss of {:.4f}, "
+        "but the autograder computed a loss of {:.4f} "
+        "based on the output of RegressionModel()".format(
+            train_loss, sanity_loss))
+
+    loss_threshold = 0.02
+    
+    if train_loss <= loss_threshold:
+        print("Your final loss is: {:f}".format(train_loss))
+        tracker.add_points(4)
+    else:
+        print("Your final loss ({:f}) must be no more than {:.4f} to receive full points for this question".format(train_loss, loss_threshold))
+
+@test('q3', points=6)
+def check_digit_classification(tracker):
+    import models
+    model = models.DigitClassificationModel()
+    dataset = backend.DigitClassificationDataset(model)
+
+    detected_parameters = None
+    
+    for batch_size in (1, 2, 4):
+        inp_x = torch.tensor(dataset.x[:batch_size], dtype=torch.float, requires_grad=True)
+        inp_y = torch.tensor(dataset.y[:batch_size], dtype=torch.float, requires_grad=True) 
+
+        loss =  model.get_loss(inp_x, inp_y)
+
+        verify_node(loss, 'tensor', (1,), "DigitClassificationModel.run()")
+
+   
+        grad_y = torch.autograd.grad(loss, inp_x, allow_unused=True, retain_graph=True)
+        grad_x = torch.autograd.grad(loss, inp_y, allow_unused=True, retain_graph=True)
+        
+        assert grad_x[0] != None, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)"
+        assert grad_y[0] != None, "Node returned from RegressionModel.get_loss() does not depend on the provided labels (y)"
+
+
+    tracker.add_points(2) # Partial credit for passing sanity checks
+
+    model.train(dataset)
+
+
+    test_logits = model.run(torch.tensor(dataset.test_images)).data
+    test_predicted = np.argmax(test_logits, axis=1).detach().numpy()
+    test_accuracy = np.mean(test_predicted == dataset.test_labels)
+
+    accuracy_threshold = 0.97
+    if test_accuracy >= accuracy_threshold:
+        print("Your final test set accuracy is: {:%}".format(test_accuracy))
+        tracker.add_points(4)
+    else:
+        print("Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question".format(test_accuracy, accuracy_threshold))
+
+@test('q4', points=7)
+def check_lang_id(tracker):
+    import models
+    model = models.LanguageIDModel()
+    dataset = backend.LanguageIDDataset(model)
+
+    detected_parameters = None
+    for batch_size, word_length in ((1, 1), (2, 1), (2, 6), (4, 8)):
+        start = dataset.dev_buckets[-1, 0]
+        end = start + batch_size
+        inp_xs, inp_y = dataset._encode(dataset.dev_x[start:end], dataset.dev_y[start:end])
+        inp_xs = torch.tensor(inp_xs[:word_length], requires_grad=True)
+
+        output_node = model.run(inp_xs)
+        verify_node(output_node, 'tensor', (batch_size, len(dataset.language_names)), "LanguageIDModel.run()")
+
+        grad = torch.autograd.grad(torch.sum(output_node), inp_xs, allow_unused=True, retain_graph=True)
+        for gradient in grad:
+            assert gradient != None, "Output returned from LanguageIDModel.run() does not depend on all of the provided inputs (xs)"
+
+        # Word length 1 does not use parameters related to transferring the
+        # hidden state across timesteps, so initial parameter detection is only
+        # run for longer words
+
+
+
+    for batch_size, word_length in ((1, 1), (2, 1), (2, 6), (4, 8)):
+        start = dataset.dev_buckets[-1, 0]
+        end = start + batch_size
+        inp_xs, inp_y = dataset._encode(dataset.dev_x[start:end], dataset.dev_y[start:end])
+        inp_xs = torch.tensor(inp_xs[:word_length], requires_grad=True)
+        loss_node = model.get_loss(inp_xs, inp_y)
+        grad = torch.autograd.grad(loss_node, inp_xs, allow_unused=True, retain_graph=True)
+        for gradient in grad:
+            assert gradient != None, "Output returned from LanguageIDModel.run() does not depend on all of the provided inputs (xs)"
+
+
+    tracker.add_points(2) # Partial credit for passing sanity checks
+
+    model.train(dataset)
+
+
+    accuracy_threshold = 0.81
+    test_accuracy = dataset.get_validation_accuracy()
+    if test_accuracy >= accuracy_threshold:
+        print("Your final test set accuracy is: {:%}".format(test_accuracy))
+        tracker.add_points(5)
+    else:
+        print("Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question".format(test_accuracy, accuracy_threshold))
+
+@test('q5', points=4)
+def check_convolution(tracker):
+    import models
+
+    model = models.DigitConvolutionalModel()
+    dataset = backend.DigitClassificationDataset2(model)
+
+    def conv2d(a, f):
+        s = f.shape + tuple(np.subtract(a.shape, f.shape) + 1)
+        strd = np.lib.stride_tricks.as_strided
+        subM = strd(a, shape = s, strides = a.strides * 2)
+        return np.einsum('ij,ijkl->kl', f, subM)
+
+    detected_parameters = None
+    
+    for batch_size in (1, 2, 4):
+        inp_x = torch.tensor(dataset[:batch_size]['x'], dtype=torch.float, requires_grad=True)
+        inp_y = torch.tensor(dataset[:batch_size]['label'], dtype=torch.float, requires_grad=True) 
+        loss =  model.get_loss(inp_x, inp_y)
+
+        verify_node(loss, 'tensor', (1,), "DigitClassificationModel.run()")
+
+   
+        grad_y = torch.autograd.grad(loss, inp_x, allow_unused=True, retain_graph=True)
+        grad_x = torch.autograd.grad(loss, inp_y, allow_unused=True, retain_graph=True)
+        
+        assert grad_x[0] != None, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)"
+        assert grad_y[0] != None, "Node returned from RegressionModel.get_loss() does not depend on the provided labels (y)"
+    
+    for matrix_size in (2, 4, 6): #Test 3 random convolutions to test convolve() function
+        weights = np.random.rand(2,2)
+        input = np.random.rand(matrix_size, matrix_size)
+        student_output = models.Convolve(torch.Tensor(input), torch.Tensor(weights))
+        actual_output = conv2d(input,weights)
+        assert np.isclose(student_output, actual_output).all(), "The convolution returned by Convolve() does not match expected output"
+
+    tracker.add_points(1) # Partial credit for testing whether convolution function works
+
+    model.train(dataset)
+
+
+    test_logits = model.run(torch.tensor(dataset.test_images)).data
+    test_predicted = np.argmax(test_logits, axis=1).detach().numpy()
+    test_accuracy = np.mean(test_predicted == dataset.test_labels)
+
+    accuracy_threshold = 0.80
+    if test_accuracy >= accuracy_threshold:
+        print("Your final test set accuracy is: {:%}".format(test_accuracy))
+        tracker.add_points(3)
+    else:
+        print("Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question".format(test_accuracy, accuracy_threshold))
+
+@test('q6', points=1)
+def check_attention(tracker):
+    import models
+    
+    for block_size in [2,4,16]:
+        layer_size = np.random.randint(2,10)
+
+        att_block = models.Attention(layer_size, block_size)
+        batch_size = np.random.randint(1,10)
+
+        input = torch.rand(batch_size, block_size, layer_size)
+
+        
+        #Define weights to manually set k, q, and v layers to
+        k_weight = torch.rand(layer_size,layer_size).reshape(layer_size,layer_size)
+        q_weight = torch.rand(layer_size,layer_size).reshape(layer_size,layer_size)
+        v_weight = torch.rand(layer_size,layer_size).reshape(layer_size,layer_size)
+
+        #Manually assign attention weights, makes autograding easier
+        with torch.no_grad():
+            att_block.k_layer.weight = torch.nn.Parameter(torch.ones((layer_size,layer_size)) * k_weight)
+            att_block.q_layer.weight = torch.nn.Parameter(torch.ones((layer_size,layer_size)).reshape(layer_size,layer_size) * q_weight)
+            att_block.v_layer.weight = torch.nn.Parameter(torch.ones((layer_size,layer_size)).reshape(layer_size,layer_size) * v_weight)
+        
+        
+        T = input.shape[1]
+        
+        expected_output = torch.matmul(att_block.k_layer(input),torch.movedim(att_block.q_layer(input),1,2))/layer_size**0.5
+
+        expected_output = expected_output.masked_fill(att_block.mask[:,:,:T,:T] == 0, float('-inf'))[0]
+        
+        output = torch.matmul(nn.functional.softmax(expected_output, dim=-1),att_block.v_layer(input))
+
+
+        assert torch.all(torch.isclose(output, att_block(input))), "The output returned by Attention() does not match expected output"
+    tracker.add_points(1)
+if __name__ == '__main__':
+    main()

proj5/machinelearning/backend.py

@@ -0,0 +1,601 @@
+import collections
+import os
+import time
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from torch import nn
+import torch
+from torch.utils.data import Dataset, DataLoader
+
+
+use_graphics = True
+
+def maybe_sleep_and_close(seconds):
+    if use_graphics and plt.get_fignums():
+        time.sleep(seconds)
+        for fignum in plt.get_fignums():
+            fig = plt.figure(fignum)
+            plt.close(fig)
+            try:
+                # This raises a TclError on some Windows machines
+                fig.canvas.start_event_loop(1e-3)
+            except:
+                pass
+
+def get_data_path(filename):
+    path = os.path.join(
+        os.path.dirname(__file__), os.pardir, "data", filename)
+    if not os.path.exists(path):
+        path = os.path.join(
+            os.path.dirname(__file__), "data", filename)
+    if not os.path.exists(path):
+        path = os.path.join(
+            os.path.dirname(__file__), filename)
+    if not os.path.exists(path):
+        raise Exception("Could not find data file: {}".format(filename))
+    return path
+
+class Custom_Dataset(Dataset):
+    def __init__(self, x, y, transform=None):
+        assert isinstance(x, np.ndarray)
+        assert isinstance(y, np.ndarray)
+        assert np.issubdtype(x.dtype, np.floating)
+        assert np.issubdtype(y.dtype, np.floating)
+        assert x.ndim == 2
+        assert y.ndim == 2
+        assert x.shape[0] == y.shape[0]
+        self.x = x
+        self.y = y
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.x)
+
+    def __getitem__(self, idx):
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+        
+        label = self.y[idx]
+        x = self.x[idx]
+         
+        sample = {'x': torch.Tensor(x), 'label': torch.Tensor(label)}
+
+        if self.transform:
+            sample = self.transform(sample)
+        
+        return sample
+    
+
+
+    def get_validation_accuracy(self):
+        raise NotImplementedError(
+            "No validation data is available for this dataset. "
+            "In this assignment, only the Digit Classification and Language "
+            "Identification datasets have validation data.")
+
+class PerceptronDataset(Custom_Dataset):
+    def __init__(self, model):
+        points = 500
+        x = np.hstack([np.random.randn(points, 2), np.ones((points, 1))])
+        y = np.where(x[:, 0] + 2 * x[:, 1] - 1 >= 0, 1.0, -1.0)
+        super().__init__(x, np.expand_dims(y, axis=1))
+
+        self.model = model
+        self.epoch = 0
+
+        if use_graphics:
+            fig, ax = plt.subplots(1, 1)
+            limits = np.array([-3.0, 3.0])
+            ax.set_xlim(limits)
+            ax.set_ylim(limits)
+            positive = ax.scatter(*x[y == 1, :-1].T, color="red", marker="+")
+            negative = ax.scatter(*x[y == -1, :-1].T, color="blue", marker="_")
+            line, = ax.plot([], [], color="black")
+            text = ax.text(0.03, 0.97, "", transform=ax.transAxes, va="top")
+            ax.legend([positive, negative], [1, -1])
+            plt.show(block=False)
+
+            self.fig = fig
+            self.limits = limits
+            self.line = line
+            self.text = text
+            self.last_update = time.time()
+    
+
+        
+    def __getitem__(self, idx):
+        self.epoch += 1
+
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+
+        x = self.x[idx]
+        y = self.y[idx]
+
+        
+
+        if use_graphics and time.time() - self.last_update > 0.01:
+            w = self.model.get_weights().data.flatten()
+            limits = self.limits
+            if w[1] != 0:
+                self.line.set_data(limits, (-w[0] * limits - w[2]) / w[1])
+            elif w[0] != 0:
+                self.line.set_data(np.full(2, -w[2] / w[0]), limits)
+            else:
+                self.line.set_data([], [])
+            self.text.set_text(
+                "epoch: {:,}\npoint: {:,}/{:,}\nweights: {}".format(
+                    self.epoch, idx * 1 + 1, len(self.x), w))
+            self.fig.canvas.draw_idle()
+            self.fig.canvas.start_event_loop(1e-3)
+            self.last_update = time.time()
+
+        return {'x': torch.tensor(x, dtype=torch.float32), 'label': torch.tensor(y, dtype=torch.float32)}
+    
+class RegressionDataset(Custom_Dataset):
+    def __init__(self, model):
+        x = np.expand_dims(np.linspace(-2 * np.pi, 2 * np.pi, num=200), axis=1)
+        np.random.RandomState(0).shuffle(x)
+        self.argsort_x = np.argsort(x.flatten())
+        y = np.sin(x)
+        super().__init__(x, y)
+
+        self.model = model
+        self.processed = 0
+
+        if use_graphics:
+            fig, ax = plt.subplots(1, 1)
+            ax.set_xlim(-2 * np.pi, 2 * np.pi)
+            ax.set_ylim(-1.4, 1.4)
+            real, = ax.plot(x[self.argsort_x], y[self.argsort_x], color="blue")
+            learned, = ax.plot([], [], color="red")
+            text = ax.text(0.03, 0.97, "", transform=ax.transAxes, va="top")
+            ax.legend([real, learned], ["real", "learned"])
+            plt.show(block=False)
+
+            self.fig = fig
+            self.learned = learned
+            self.text = text
+            self.last_update = time.time()
+
+    def __len__(self):
+        return len(self.x)
+
+    def __getitem__(self, idx):
+
+        data = super().__getitem__(idx)
+
+        x = data['x']
+        y = data['label']
+
+        self.processed += 1
+
+        if use_graphics and time.time() - self.last_update > 0.1:
+            predicted = self.model(torch.tensor(self.x, dtype=torch.float32)).data
+            loss = self.model.get_loss(
+                x, y).data
+            self.learned.set_data(self.x[self.argsort_x], predicted[self.argsort_x])
+            self.text.set_text("processed: {:,}\nloss: {:.6f}".format(
+                self.processed, loss))
+            self.fig.canvas.draw_idle()
+            self.fig.canvas.start_event_loop(1e-3)
+            self.last_update = time.time()
+        
+        return {'x': x, 'label': y}
+
+class DigitClassificationDataset(Custom_Dataset):
+    def __init__(self, model):
+        mnist_path = get_data_path("mnist.npz")
+
+        with np.load(mnist_path) as data:
+            train_images = data["train_images"]
+            train_labels = data["train_labels"]
+            test_images = data["test_images"]
+            test_labels = data["test_labels"]
+            assert len(train_images) == len(train_labels) == 60000
+            assert len(test_images) == len(test_labels) == 10000
+            self.dev_images = test_images[0::2]
+            self.dev_labels = test_labels[0::2]
+            self.test_images = test_images[1::2]
+            self.test_labels = test_labels[1::2]
+
+        train_labels_one_hot = np.zeros((len(train_images), 10))
+        train_labels_one_hot[range(len(train_images)), train_labels] = 1
+
+        super().__init__(train_images, train_labels_one_hot)
+
+        self.model = model
+        self.epoch = 0
+        self.num_items = 0
+
+        if use_graphics:
+            self.current_accuracy = None
+            width = 20  # Width of each row expressed as a multiple of image width
+            samples = 100  # Number of images to display per label
+            fig = plt.figure()
+            ax = {}
+            images = collections.defaultdict(list)
+            texts = collections.defaultdict(list)
+            for i in reversed(range(10)):
+                ax[i] = plt.subplot2grid((30, 1), (3 * i, 0), 2, 1,
+                                         sharex=ax.get(9))
+                plt.setp(ax[i].get_xticklabels(), visible=i == 9)
+                ax[i].set_yticks([])
+                ax[i].text(-0.03, 0.5, i, transform=ax[i].transAxes,
+                           va="center")
+                ax[i].set_xlim(0, 28 * width)
+                ax[i].set_ylim(0, 28)
+                for j in range(samples):
+                    images[i].append(ax[i].imshow(
+                        np.zeros((28, 28)), vmin=0, vmax=1, cmap="Greens",
+                        alpha=0.3))
+                    texts[i].append(ax[i].text(
+                        0, 0, "", ha="center", va="top", fontsize="smaller"))
+            ax[9].set_xticks(np.linspace(0, 28 * width, 11))
+            ax[9].set_xticklabels(
+                ["{:.1f}".format(num) for num in np.linspace(0, 1, 11)])
+            ax[9].tick_params(axis="x", pad=16)
+            ax[9].set_xlabel("Probability of Correct Label")
+            status = ax[0].text(
+                0.5, 1.5, "", transform=ax[0].transAxes, ha="center",
+                va="bottom")
+            plt.show(block=False)
+
+            self.width = width
+            self.samples = samples
+            self.fig = fig
+            self.images = images
+            self.texts = texts
+            self.status = status
+            self.last_update = time.time()
+
+
+    def __getitem__(self, idx):
+        
+
+        data = super().__getitem__(idx)
+        
+        x = data['x']
+        y = data['label']
+
+        if use_graphics and time.time() - self.last_update > 1:
+            dev_logits = self.model.run(torch.tensor(self.dev_images)).data
+            dev_predicted = np.argmax(dev_logits, axis=1).detach().numpy()
+            dev_probs = np.exp(nn.functional.log_softmax(dev_logits))
+
+            dev_accuracy = np.mean(dev_predicted == self.dev_labels)
+            self.status.set_text(
+                    "validation accuracy: "
+                    "{:.2%}".format(
+                        dev_accuracy))
+            for i in range(10):
+                predicted = dev_predicted[self.dev_labels == i]
+                probs = dev_probs[self.dev_labels == i][:, i]
+                linspace = np.linspace(
+                    0, len(probs) - 1, self.samples).astype(int)
+                indices = probs.argsort()[linspace]
+                for j, (prob, image) in enumerate(zip(
+                        probs[indices],
+                        self.dev_images[self.dev_labels == i][indices])):
+                    self.images[i][j].set_data(image.reshape((28, 28)))
+                    left = prob * (self.width - 1) * 28
+                    if predicted[indices[j]] == i:
+                        self.images[i][j].set_cmap("Greens")
+                        self.texts[i][j].set_text("")
+                    else:
+                        self.images[i][j].set_cmap("Reds")
+                        self.texts[i][j].set_text(predicted[indices[j]])
+                        self.texts[i][j].set_x(left + 14)
+                    self.images[i][j].set_extent([left, left + 28, 0, 28])
+            self.fig.canvas.draw_idle()
+            self.fig.canvas.start_event_loop(1e-3)
+            self.last_update = time.time()
+        
+            if(self.num_items == len(self.x)):
+                self.current_accuracy = self.num_right_items/len(self.x)
+                self.num_right_items = 0
+                self.epoch += 1
+
+        return {'x': x, 'label': y}
+
+    def get_validation_accuracy(self):
+        dev_logits = self.model.run(torch.tensor(self.dev_images)).data
+        dev_predicted = np.argmax(dev_logits, axis=1).detach().numpy()
+        dev_probs = np.exp(nn.functional.log_softmax(dev_logits))
+
+        dev_accuracy = np.mean(dev_predicted == self.dev_labels)
+        return dev_accuracy
+
+class LanguageIDDataset(Custom_Dataset):
+    def __init__(self, model):
+        self.model = model
+
+        data_path = get_data_path("lang_id.npz")
+
+        with np.load(data_path) as data:
+            self.chars = data['chars']
+            self.language_codes = data['language_codes']
+            self.language_names = data['language_names']
+            self.train_x = data['train_x']
+            self.train_y = data['train_y']
+            self.train_buckets = data['train_buckets']
+            self.dev_x = data['dev_x']
+            self.dev_y = data['dev_y']
+            self.dev_buckets = data['dev_buckets']
+            self.test_x = data['test_x']
+            self.test_y = data['test_y']
+            self.test_buckets = data['test_buckets']
+
+        self.epoch = 0
+        self.bucket_weights = self.train_buckets[:,1] - self.train_buckets[:,0]
+        self.bucket_weights = self.bucket_weights / float(self.bucket_weights.sum())
+
+        self.chars_print = self.chars
+        try:
+            print(u"Alphabet: {}".format(u"".join(self.chars)))
+        except UnicodeEncodeError:
+            self.chars_print = "abcdefghijklmnopqrstuvwxyzaaeeeeiinoouuacelnszz"
+            print("Alphabet: " + self.chars_print)
+            self.chars_print = list(self.chars_print)
+            print("""
+NOTE: Your terminal does not appear to support printing Unicode characters.
+For the purposes of printing to the terminal, some of the letters in the
+alphabet above have been substituted with ASCII symbols.""".strip())
+        print("")
+
+        # Select some examples to spotlight in the monitoring phase (3 per language)
+        spotlight_idxs = []
+        for i in range(len(self.language_names)):
+            idxs_lang_i = np.nonzero(self.dev_y == i)[0]
+            idxs_lang_i = np.random.choice(idxs_lang_i, size=3, replace=False)
+            spotlight_idxs.extend(list(idxs_lang_i))
+        self.spotlight_idxs = np.array(spotlight_idxs, dtype=int)
+
+        # Templates for printing updates as training progresses
+        max_word_len = self.dev_x.shape[1]
+        max_lang_len = max([len(x) for x in self.language_names])
+
+        self.predicted_template = u"Pred: {:<NUM}".replace('NUM',
+            str(max_lang_len))
+
+        self.word_template = u"  "
+        self.word_template += u"{:<NUM} ".replace('NUM', str(max_word_len))
+        self.word_template += u"{:<NUM} ({:6.1%})".replace('NUM', str(max_lang_len))
+        self.word_template += u" {:<NUM} ".replace('NUM',
+            str(max_lang_len + len('Pred: ')))
+        for i in range(len(self.language_names)):
+            self.word_template += u"|{}".format(self.language_codes[i])
+            self.word_template += "{probs[" + str(i) + "]:4.0%}"
+
+        self.last_update = time.time()
+
+    def __len__(self):
+        return len(self.train_x)
+    
+    def _encode(self, inp_x, inp_y):
+        xs = []
+        for i in range(inp_x.shape[1]):
+
+            if np.all(np.array(inp_x[:,i])  == -1):
+                break
+            assert not np.any(np.array(inp_x[:,i]) == -1), (
+                "Please report this error in the project: batching by length was done incorrectly in the provided code")
+            x = np.eye(len(self.chars))[np.array(inp_x[:,i], dtype=int)]
+            xs.append(x)
+        y = np.eye(len(self.language_names))[inp_y]
+        j = [[0 for j in range(47)]]
+        
+        if(len(inp_x) == 1):
+            return torch.nn.functional.pad(torch.tensor(xs, dtype=torch.float),(0,0,0,0,0,10 - len(xs))), torch.tensor(y, dtype=torch.float)
+
+        return torch.tensor(xs, dtype=torch.float), torch.tensor(y, dtype=torch.float)
+
+    def _softmax(self, x):
+        exp = np.exp(x - np.max(x, axis=-1, keepdims=True))
+        return exp / np.sum(exp, axis=-1, keepdims=True)
+
+    def _predict(self, split='test'):
+        if split == 'dev':
+            data_x = self.dev_x
+            data_y = self.dev_y
+            buckets = self.dev_buckets
+        else:
+            data_x = self.test_x
+            data_y = self.test_y
+            buckets = self.test_buckets
+
+        all_predicted = []
+        all_correct = []
+        for bucket_id in range(buckets.shape[0]):
+            start, end = buckets[bucket_id]
+            xs, y = self._encode(data_x[start:end], data_y[start:end])
+            predicted = self.model.run(xs)
+
+            all_predicted.extend(list(predicted.data))
+            all_correct.extend(list(data_y[start:end]))
+        sftmax = nn.Softmax()
+        all_predicted_probs = [sftmax(torch.tensor(i)) for i in all_predicted]
+
+        all_predicted = [i.argmax() for i in all_predicted_probs]
+        all_correct = np.asarray(all_correct)
+
+        return all_predicted_probs, all_predicted, all_correct
+
+    def __getitem__(self, idx):
+
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+
+        
+        ret = self._encode(self.train_x[idx:idx+1], self.train_y[idx:idx+1])
+        return {'x': torch.squeeze(ret[0]), 'label': torch.squeeze(ret[1])}
+
+    def get_validation_accuracy(self):
+        dev_predicted_probs, dev_predicted, dev_correct = self._predict()
+        dev_accuracy = np.mean(dev_predicted == dev_correct)
+        return dev_accuracy
+    
+    def collate(self, batch):
+        '''
+        Padds batch of variable length
+
+
+        '''
+        ## get sequence lengths
+        lengths = torch.tensor([ t['x'].shape[0] for t in batch ])
+        ## padd
+        batch_x = [ torch.Tensor(t['x']) for t in batch ]
+        batch_y = [ torch.Tensor(t['labels']) for t in batch ]
+        return {'x':batch_x,'label':batch_y}
+
+
+class DigitClassificationDataset2(Custom_Dataset):
+    def __init__(self, model):
+        mnist_path = get_data_path("mnist.npz")
+        training_size = 200
+        test_size = 100
+        with np.load(mnist_path) as data:
+            train_images = data["train_images"][:training_size]
+            train_labels = data["train_labels"][:training_size]
+            test_images = data["train_images"][:test_size]
+            test_labels = data["train_labels"][:test_size]
+            assert len(train_images) == len(train_labels) == training_size
+            assert len(test_images) == len(test_labels) == test_size
+            self.dev_images = test_images[0::2]
+            self.dev_labels = test_labels[0::2]
+            self.test_images = test_images[1::2]
+            self.test_labels = test_labels[1::2]
+
+        train_labels_one_hot = np.zeros((len(train_images), 10))
+        train_labels_one_hot[range(len(train_images)), train_labels] = 1
+
+        super().__init__(train_images, train_labels_one_hot)
+
+        self.model = model
+        self.epoch = 0
+        self.num_items = 0
+
+        if use_graphics:
+            self.current_accuracy = None
+            width = 20  # Width of each row expressed as a multiple of image width
+            samples = 100  # Number of images to display per label
+            fig = plt.figure()
+            ax = {}
+            images = collections.defaultdict(list)
+            texts = collections.defaultdict(list)
+            for i in reversed(range(10)):
+                ax[i] = plt.subplot2grid((30, 1), (3 * i, 0), 2, 1,
+                                         sharex=ax.get(9))
+                plt.setp(ax[i].get_xticklabels(), visible=i == 9)
+                ax[i].set_yticks([])
+                ax[i].text(-0.03, 0.5, i, transform=ax[i].transAxes,
+                           va="center")
+                ax[i].set_xlim(0, 28 * width)
+                ax[i].set_ylim(0, 28)
+                for j in range(samples):
+                    images[i].append(ax[i].imshow(
+                        np.zeros((28, 28)), vmin=0, vmax=1, cmap="Greens",
+                        alpha=0.3))
+                    texts[i].append(ax[i].text(
+                        0, 0, "", ha="center", va="top", fontsize="smaller"))
+            ax[9].set_xticks(np.linspace(0, 28 * width, 11))
+            ax[9].set_xticklabels(
+                ["{:.1f}".format(num) for num in np.linspace(0, 1, 11)])
+            ax[9].tick_params(axis="x", pad=16)
+            ax[9].set_xlabel("Probability of Correct Label")
+            status = ax[0].text(
+                0.5, 1.5, "", transform=ax[0].transAxes, ha="center",
+                va="bottom")
+            plt.show(block=False)
+
+            self.width = width
+            self.samples = samples
+            self.fig = fig
+            self.images = images
+            self.texts = texts
+            self.status = status
+            self.last_update = time.time()
+
+
+    def __getitem__(self, idx):
+        
+
+        data = super().__getitem__(idx)
+        
+        x = data['x']
+        y = data['label']
+
+        if use_graphics and time.time() - self.last_update > 1:
+            dev_logits = self.model.run(torch.tensor(self.dev_images)).data
+            dev_predicted = np.argmax(dev_logits, axis=1).detach().numpy()
+            dev_probs = np.exp(nn.functional.log_softmax(dev_logits))
+
+            dev_accuracy = np.mean(dev_predicted == self.dev_labels)
+            self.status.set_text(
+                    "validation accuracy: "
+                    "{:.2%}".format(
+                        dev_accuracy))
+            for i in range(10):
+                predicted = dev_predicted[self.dev_labels == i]
+                probs = dev_probs[self.dev_labels == i][:, i]
+                linspace = np.linspace(
+                    0, len(probs) - 1, self.samples).astype(int)
+                indices = probs.argsort()[linspace]
+                for j, (prob, image) in enumerate(zip(
+                        probs[indices],
+                        self.dev_images[self.dev_labels == i][indices])):
+                    self.images[i][j].set_data(image.reshape((28, 28)))
+                    left = prob * (self.width - 1) * 28
+                    if predicted[indices[j]] == i:
+                        self.images[i][j].set_cmap("Greens")
+                        self.texts[i][j].set_text("")
+                    else:
+                        self.images[i][j].set_cmap("Reds")
+                        self.texts[i][j].set_text(predicted[indices[j]])
+                        self.texts[i][j].set_x(left + 14)
+                    self.images[i][j].set_extent([left, left + 28, 0, 28])
+            self.fig.canvas.draw_idle()
+            self.fig.canvas.start_event_loop(1e-3)
+            self.last_update = time.time()
+        
+            if(self.num_items == len(self.x)):
+                self.current_accuracy = self.num_right_items/len(self.x)
+                self.num_right_items = 0
+                self.epoch += 1
+
+        return {'x': x, 'label': y}
+
+    def get_validation_accuracy(self):
+        dev_logits = self.model.run(torch.tensor(self.dev_images)).data
+        dev_predicted = np.argmax(dev_logits, axis=1).detach().numpy()
+        dev_probs = np.exp(nn.functional.log_softmax(dev_logits))
+
+        dev_accuracy = np.mean(dev_predicted == self.dev_labels)
+        return dev_accuracy
+
+
+    
+def main():
+    import models
+    model = models.PerceptronModel(3)
+    dataset = PerceptronDataset(model)
+    model.train(dataset)
+
+    model = models.RegressionModel()
+    dataset = RegressionDataset(model)
+    model.train(dataset)
+
+    model = models.DigitClassificationModel()
+    dataset = DigitClassificationDataset(model)
+    model.train(dataset)
+
+    model = models.LanguageIDModel()
+    dataset = LanguageIDDataset(model)
+    model.train(dataset)
+
+if __name__ == "__main__":
+    main()
+

proj5/machinelearning/chargpt.py

@@ -0,0 +1,112 @@
+"""
+The MIT License (MIT) Copyright (c) 2020 Andrej Karpathy
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+
+Modified version of Andrej Karpathy's "mingpt" repo found here: https://github.com/karpathy/minGPT
+"""
+
+
+import torch
+from torch.utils.data import Dataset
+from torch.utils.data.dataloader import DataLoader
+
+from gpt_model import Character_GPT
+
+
+
+class CharDataset(Dataset):
+    def __init__(self, data):
+        #Configure block size
+        self.block_size = 10
+
+        #Define possible characters, and create mapping from number to character
+        chars = sorted(list(set(data)))
+        self.index2char = { i:ch for i,ch in enumerate(chars) }
+        self.char2index = { ch:i for i,ch in enumerate(chars) }
+        vocab_size = len(chars)
+        self.vocab_size = vocab_size
+        self.data = data
+
+    def __len__(self):
+        return len(self.data) - self.block_size
+
+    def __getitem__(self, idx):
+        # grab a chunk of (block_size + 1) characters from the data
+        chunk = self.data[idx:idx + self.block_size + 1]
+        # encode every character to an integer
+        dix = [self.char2index[s] for s in chunk]
+        # return as tensors
+        x = torch.tensor(dix[:-1], dtype=torch.long)
+        y = torch.tensor(dix[1:],dtype=torch.long)
+        return x, y
+
+
+
+def train_single_iteration(model, data_iter):
+    try:
+        batch = next(data_iter)
+    except StopIteration:
+        data_iter = iter(train_loader)
+        batch = next(data_iter)
+    batch = [t for t in batch]
+    x, y = batch
+
+    # forward the model
+    loss = model.get_loss(x, y)
+
+    # backprop and update the parameters
+    model.zero_grad(set_to_none=True)
+    loss.backward()
+   
+    optimizer.step()
+
+ 
+if __name__ == '__main__':
+    #Variables to configure
+    learning_rate = 0.0004
+    num_iterations = 10000
+    batch_size = 500 
+    checkpoint = 100 #How often to print out results of the model during training
+    context = "Pacman" #Generative prompt
+    layer_size = 100
+    n_layer = 6 #How many transformer blocks to have
+
+
+    # construct the training dataset
+    text = open('input.txt', 'r').read() 
+    train_dataset = CharDataset(text)
+
+    # setup the dataloader
+    train_loader = DataLoader(
+        train_dataset,
+        sampler=torch.utils.data.RandomSampler(train_dataset, replacement=True, num_samples=int(1e10)),
+        shuffle=False,
+        pin_memory=True,
+        batch_size=batch_size,
+        num_workers=1,
+    )
+
+    train_iterations = iter(train_loader)
+
+    #set up model and optimizer
+    model = Character_GPT(train_dataset.block_size, n_embd=layer_size, n_layer=n_layer, vocab_size=train_dataset.vocab_size)
+    optimizer = torch.optim.Adam(model.parameters(), lr=0.00004)
+
+
+    for i in range(num_iterations):
+        train_single_iteration(model, train_iterations)
+        
+        if i % checkpoint == 0:
+            with torch.no_grad():
+                print("Iteration: " + str(i) + "\n")
+                
+                # sample from the model...
+                print("Prompt: " + context)
+                print("Generated result: ")
+                x = torch.tensor([train_dataset.char2index[s] for s in context])[None,...]
+                y = model.generate(x, 500)[0]
+                completion = ''.join([train_dataset.index2char[int(i)] for i in y])
+                print(completion + "\n")
+   

proj5/machinelearning/gpt_model.py

@@ -0,0 +1,98 @@
+"""
+The MIT License (MIT) Copyright (c) 2020 Andrej Karpathy
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+
+References:
+1) "minGPT" implemented by Andrej Karpathy
+https://github.com/karpathy/minGPT
+2) the official GPT-2 TensorFlow implementation released by OpenAI:
+https://github.com/openai/gpt-2/blob/master/src/model.py
+3) huggingface/transformers PyTorch implementation:
+https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt2/modeling_gpt2.py
+"""
+
+
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+from models import Attention
+
+
+class Transformer_Block(nn.Module):
+    """
+    This class builds the basic transformer block.
+    """
+    def __init__(self, n_embd, block_size):
+        super().__init__()
+
+        self.attn_block = Attention(n_embd, block_size)
+        self.norm_1 = nn.LayerNorm(n_embd)
+        self.linear_1 = nn.Linear(n_embd, n_embd)
+        self.norm_2 = nn.LayerNorm(n_embd)
+
+
+    def forward(self, x):
+        """YOUR CODE HERE"""
+
+       
+
+class Character_GPT(nn.Module):
+   
+    def __init__(self, block_size, n_embd, n_layer, vocab_size):
+        super().__init__()
+        self.block_size = block_size
+        self.embed = nn.Embedding(vocab_size, n_embd) #Embedding layer, think of this as similar to a linear layer
+
+        
+        self.transformer_blocks = nn.ModuleList([Transformer_Block(n_embd, block_size) for _ in range(n_layer)]) #You can treat this as a python list
+        self.norm = nn.LayerNorm(n_embd) #Normalization Layer
+        self.output_layer = nn.Linear(n_embd, vocab_size, bias=False)
+
+
+
+    def get_loss(self, input, target):
+        output = self(input)
+        return F.cross_entropy(output.view(-1, output.size(-1)), target.view(-1), ignore_index=-1)
+        
+    def forward(self, input):
+        """
+        This function should take in an input representing a sequence of characters, and output
+        an array representing the likelihood of any character appearing next.
+
+        All necessary layers have been initialized for you in the __init__() function, you should pay special
+        attention to the self.transformer_blocks variable. Since we have multiple transformer blocks in our
+        final model, you will have to pass the input through every object in this list.
+        """
+        b, t = input.size()
+        assert t <= self.block_size, f"Cannot forward sequence of length {t}, block size is only {self.block_size}"
+
+        """YOUR CODE HERE"""
+
+
+    @torch.no_grad()
+    def generate(self, idx, max_new_tokens):
+        """
+        Take a conditioning sequence of indices idx (LongTensor of shape (b,t)) and complete
+        the sequence max_new_tokens times, feeding the predictions back into the model each time.
+        """
+        for _ in range(max_new_tokens):
+            # if the sequence context is growing too long we must crop it at block_size
+            idx_cond = idx if idx.size(1) <= self.block_size else idx[:, -self.block_size:]
+            # forward the model to get the logits for the index in the sequence
+            logits = self(idx_cond)
+            # pluck the logits at the final step and scale by desired temperature
+            logits = logits[:, -1, :]
+            # optionally crop the logits to only the top k options
+
+            # apply softmax to convert logits to (normalized) probabilities
+            probs = F.softmax(logits, dim=-1)
+            # either sample from the distribution or take the most likely element
+            
+            idx_next = torch.multinomial(probs, num_samples=1)
+            
+            # append sampled index to the running sequence and continue
+            idx = torch.cat((idx, idx_next), dim=1)
+
+        return idx

proj5/machinelearning/models.py

@@ -0,0 +1,429 @@
+from torch import no_grad, stack
+from torch.utils.data import DataLoader
+from torch.nn import Module
+
+
+"""
+Functions you should use.
+Please avoid importing any other functions or modules.
+Your code will not pass if the gradescope autograder detects any changed imports
+"""
+import torch
+from torch.nn import Parameter, Linear
+from torch import optim, tensor, tensordot, ones, matmul
+from torch.nn.functional import cross_entropy, relu, mse_loss, softmax
+from torch import movedim
+
+
+class PerceptronModel(Module):
+    def __init__(self, dimensions):
+        """
+        Initialize a new Perceptron instance.
+
+        A perceptron classifies data points as either belonging to a particular
+        class (+1) or not (-1). `dimensions` is the dimensionality of the data.
+        For example, dimensions=2 would mean that the perceptron must classify
+        2D points.
+
+        In order for our autograder to detect your weight, initialize it as a 
+        pytorch Parameter object as follows:
+
+        Parameter(weight_vector)
+
+        where weight_vector is a pytorch Tensor of dimension 'dimensions'
+
+        
+        Hint: You can use ones(dim) to create a tensor of dimension dim.
+        """
+        super(PerceptronModel, self).__init__()
+        
+        "*** YOUR CODE HERE ***"
+        
+
+    def get_weights(self):
+        """
+        Return a Parameter instance with the current weights of the perceptron.
+        """
+        return self.w
+
+    def run(self, x):
+        """
+        Calculates the score assigned by the perceptron to a data point x.
+
+        Inputs:
+            x: a node with shape (1 x dimensions)
+        Returns: a node containing a single number (the score)
+
+        The pytorch function `tensordot` may be helpful here.
+        """
+        "*** YOUR CODE HERE ***"
+        
+
+    def get_prediction(self, x):
+        """
+        Calculates the predicted class for a single data point `x`.
+
+        Returns: 1 or -1
+        """
+        "*** YOUR CODE HERE ***"
+
+
+
+    def train(self, dataset):
+        """
+        Train the perceptron until convergence.
+        You can iterate through DataLoader in order to 
+        retrieve all the batches you need to train on.
+
+        Each sample in the dataloader is in the form {'x': features, 'label': label} where label
+        is the item we need to predict based off of its features.
+        """        
+        with no_grad():
+            dataloader = DataLoader(dataset, batch_size=1, shuffle=True)
+            "*** YOUR CODE HERE ***"
+
+
+
+class RegressionModel(Module):
+    """
+    A neural network model for approximating a function that maps from real
+    numbers to real numbers. The network should be sufficiently large to be able
+    to approximate sin(x) on the interval [-2pi, 2pi] to reasonable precision.
+    """
+    def __init__(self):
+        # Initialize your model parameters here
+        "*** YOUR CODE HERE ***"
+        super().__init__()
+
+
+
+    def forward(self, x):
+        """
+        Runs the model for a batch of examples.
+
+        Inputs:
+            x: a node with shape (batch_size x 1)
+        Returns:
+            A node with shape (batch_size x 1) containing predicted y-values
+        """
+        "*** YOUR CODE HERE ***"
+
+    
+    def get_loss(self, x, y):
+        """
+        Computes the loss for a batch of examples.
+
+        Inputs:
+            x: a node with shape (batch_size x 1)
+            y: a node with shape (batch_size x 1), containing the true y-values
+                to be used for training
+        Returns: a tensor of size 1 containing the loss
+        """
+        "*** YOUR CODE HERE ***"
+ 
+        
+
+    def train(self, dataset):
+        """
+        Trains the model.
+
+        In order to create batches, create a DataLoader object and pass in `dataset` as well as your required 
+        batch size. You can look at PerceptronModel as a guideline for how you should implement the DataLoader
+
+        Each sample in the dataloader object will be in the form {'x': features, 'label': label} where label
+        is the item we need to predict based off of its features.
+
+        Inputs:
+            dataset: a PyTorch dataset object containing data to be trained on
+            
+        """
+        "*** YOUR CODE HERE ***"
+
+            
+
+
+
+
+
+
+
+class DigitClassificationModel(Module):
+    """
+    A model for handwritten digit classification using the MNIST dataset.
+
+    Each handwritten digit is a 28x28 pixel grayscale image, which is flattened
+    into a 784-dimensional vector for the purposes of this model. Each entry in
+    the vector is a floating point number between 0 and 1.
+
+    The goal is to sort each digit into one of 10 classes (number 0 through 9).
+
+    (See RegressionModel for more information about the APIs of different
+    methods here. We recommend that you implement the RegressionModel before
+    working on this part of the project.)
+    """
+    def __init__(self):
+        # Initialize your model parameters here
+        super().__init__()
+        input_size = 28 * 28
+        output_size = 10
+        "*** YOUR CODE HERE ***"
+
+
+
+
+    def run(self, x):
+        """
+        Runs the model for a batch of examples.
+
+        Your model should predict a node with shape (batch_size x 10),
+        containing scores. Higher scores correspond to greater probability of
+        the image belonging to a particular class.
+
+        Inputs:
+            x: a tensor with shape (batch_size x 784)
+        Output:
+            A node with shape (batch_size x 10) containing predicted scores
+                (also called logits)
+        """
+        """ YOUR CODE HERE """
+
+ 
+
+    def get_loss(self, x, y):
+        """
+        Computes the loss for a batch of examples.
+
+        The correct labels `y` are represented as a tensor with shape
+        (batch_size x 10). Each row is a one-hot vector encoding the correct
+        digit class (0-9).
+
+        Inputs:
+            x: a node with shape (batch_size x 784)
+            y: a node with shape (batch_size x 10)
+        Returns: a loss tensor
+        """
+        """ YOUR CODE HERE """
+
+    
+        
+
+    def train(self, dataset):
+        """
+        Trains the model.
+        """
+        """ YOUR CODE HERE """
+
+
+
+class LanguageIDModel(Module):
+    """
+    A model for language identification at a single-word granularity.
+
+    (See RegressionModel for more information about the APIs of different
+    methods here. We recommend that you implement the RegressionModel before
+    working on this part of the project.)
+    """
+    def __init__(self):
+        # Our dataset contains words from five different languages, and the
+        # combined alphabets of the five languages contain a total of 47 unique
+        # characters.
+        # You can refer to self.num_chars or len(self.languages) in your code
+        self.num_chars = 47
+        self.languages = ["English", "Spanish", "Finnish", "Dutch", "Polish"]
+        super(LanguageIDModel, self).__init__()
+        "*** YOUR CODE HERE ***"
+
+
+    def run(self, xs):
+        """
+        Runs the model for a batch of examples.
+
+        Although words have different lengths, our data processing guarantees
+        that within a single batch, all words will be of the same length (L).
+
+        Here `xs` will be a list of length L. Each element of `xs` will be a
+        tensor with shape (batch_size x self.num_chars), where every row in the
+        array is a one-hot vector encoding of a character. For example, if we
+        have a batch of 8 three-letter words where the last word is "cat", then
+        xs[1] will be a tensor that contains a 1 at position (7, 0). Here the
+        index 7 reflects the fact that "cat" is the last word in the batch, and
+        the index 0 reflects the fact that the letter "a" is the inital (0th)
+        letter of our combined alphabet for this task.
+
+        Your model should use a Recurrent Neural Network to summarize the list
+        `xs` into a single tensor of shape (batch_size x hidden_size), for your
+        choice of hidden_size. It should then calculate a tensor of shape
+        (batch_size x 5) containing scores, where higher scores correspond to
+        greater probability of the word originating from a particular language.
+
+        Inputs:
+            xs: a list with L elements (one per character), where each element
+                is a node with shape (batch_size x self.num_chars)
+        Returns:
+            A node with shape (batch_size x 5) containing predicted scores
+                (also called logits)
+        """
+        "*** YOUR CODE HERE ***"
+
+    
+    def get_loss(self, xs, y):
+        """
+        Computes the loss for a batch of examples.
+
+        The correct labels `y` are represented as a node with shape
+        (batch_size x 5). Each row is a one-hot vector encoding the correct
+        language.
+
+        Inputs:
+            xs: a list with L elements (one per character), where each element
+                is a node with shape (batch_size x self.num_chars)
+            y: a node with shape (batch_size x 5)
+        Returns: a loss node
+        """
+        "*** YOUR CODE HERE ***"
+        
+
+    def train(self, dataset):
+        """
+        Trains the model.
+
+        Note that when you iterate through dataloader, each batch will returned as its own vector in the form
+        (batch_size x length of word x self.num_chars). However, in order to run multiple samples at the same time,
+        get_loss() and run() expect each batch to be in the form (length of word x batch_size x self.num_chars), meaning
+        that you need to switch the first two dimensions of every batch. This can be done with the movedim() function 
+        as follows:
+
+        movedim(input_vector, initial_dimension_position, final_dimension_position)
+
+        For more information, look at the pytorch documentation of torch.movedim()
+        """
+        "*** YOUR CODE HERE ***"
+
+        
+
+def Convolve(input: tensor, weight: tensor):
+    """
+    Acts as a convolution layer by applying a 2d convolution with the given inputs and weights.
+    DO NOT import any pytorch methods to directly do this, the convolution must be done with only the functions
+    already imported.
+
+    There are multiple ways to complete this function. One possible solution would be to use 'tensordot'.
+    If you would like to index a tensor, you can do it as such:
+
+    tensor[y:y+height, x:x+width]
+
+    This returns a subtensor who's first element is tensor[y,x] and has height 'height, and width 'width'
+    """
+    input_tensor_dimensions = input.shape
+    weight_dimensions = weight.shape
+    Output_Tensor = tensor(())
+    "*** YOUR CODE HERE ***"
+
+    
+    "*** End Code ***"
+    return Output_Tensor
+
+
+
+class DigitConvolutionalModel(Module):
+    """
+    A model for handwritten digit classification using the MNIST dataset.
+
+    This class is a convolutational model which has already been trained on MNIST.
+    if Convolve() has been correctly implemented, this model should be able to achieve a high accuracy
+    on the mnist dataset given the pretrained weights.
+
+    Note that this class looks different from a standard pytorch model since we don't need to train it
+    as it will be run on preset weights.
+    """
+    
+
+    def __init__(self):
+        # Initialize your model parameters here
+        super().__init__()
+        output_size = 10
+
+        self.convolution_weights = Parameter(ones((3, 3)))
+        """ YOUR CODE HERE """
+
+
+
+
+    def run(self, x):
+        return self(x)
+ 
+    def forward(self, x):
+        """
+        The convolutional layer is already applied, and the output is flattened for you. You should treat x as
+        a regular 1-dimentional datapoint now, similar to the previous questions.
+        """
+        x = x.reshape(len(x), 28, 28)
+        x = stack(list(map(lambda sample: Convolve(sample, self.convolution_weights), x)))
+        x = x.flatten(start_dim=1)
+        """ YOUR CODE HERE """
+
+
+    def get_loss(self, x, y):
+        """
+        Computes the loss for a batch of examples.
+
+        The correct labels `y` are represented as a tensor with shape
+        (batch_size x 10). Each row is a one-hot vector encoding the correct
+        digit class (0-9).
+
+        Inputs:
+            x: a node with shape (batch_size x 784)
+            y: a node with shape (batch_size x 10)
+        Returns: a loss tensor
+        """
+        """ YOUR CODE HERE """
+
+     
+        
+
+    def train(self, dataset):
+        """
+        Trains the model.
+        """
+        """ YOUR CODE HERE """
+
+
+
+class Attention(Module):
+    def __init__(self, layer_size, block_size):
+        super().__init__()
+        """
+        All the layers you should use are defined here.
+
+        In order to pass the autograder, make sure each linear layer matches up with their corresponding matrix,
+        ie: use self.k_layer to generate the K matrix.
+        """
+        self.k_layer = Linear(layer_size, layer_size)
+        self.q_layer = Linear(layer_size, layer_size)
+        self.v_layer = Linear(layer_size,layer_size)
+
+        #Masking part of attention layer
+        self.register_buffer("mask", torch.tril(torch.ones(block_size, block_size))
+                                     .view(1, 1, block_size, block_size))
+       
+        self.layer_size = layer_size
+
+
+    def forward(self, input):
+        """
+        Applies the attention mechanism to input. All necessary layers have 
+        been defined in __init__()
+
+        In order to apply the causal mask to a given matrix M, you should update
+        it as such:
+    
+        M = M.masked_fill(self.mask[:,:,:T,:T] == 0, float('-inf'))[0]
+
+        For the softmax activation, it should be applied to the last dimension of the input,
+        Take a look at the "dim" argument of torch.nn.functional.softmax to figure out how to do this.
+        """
+        B, T, C = input.size()
+
+        """YOUR CODE HERE"""
+
+     

proj5/machinelearning/proj5-readme.txt

@@ -0,0 +1,390 @@
+Introduction
+
+This project will be an introduction to machine learning; you will build a neural network to classify digits, and more! You can download the project as a zip file here: zip archive.
+Files you'll edit:
+models.py 	Perceptron and neural network models for a variety of applications.
+Supporting files you can ignore:
+autograder.py 	Project autograder.
+backend.py 	Backend code for various machine learning tasks.
+data 	Datasets for digit classification and language identification.
+
+Files to Edit and Submit: You will fill in portions of models.py during the assignment. Once you have completed the assignment, you will submit these files to Gradescope (for instance, you can upload all .py files in the folder). Please do not change the other files in this distribution.
+
+Evaluation: Your code will be autograded for technical correctness. Please do not change the names of any provided functions or classes within the code, or you will wreak havoc on the autograder. However, the correctness of your implementation – not the autograder’s judgements – will be the final judge of your score. If necessary, we will review and grade assignments individually to ensure that you receive due credit for your work.
+
+Academic Dishonesty: We will be checking your code against other submissions in the class for logical redundancy. If you copy someone else’s code and submit it with minor changes, we will know. These cheat detectors are quite hard to fool, so please don’t try. We trust you all to submit your own work only; please don’t let us down. If you do, we will pursue the strongest consequences available to us.
+
+Getting Help: You are not alone! If you find yourself stuck on something, contact the course staff for help. Office hours, section, and the discussion forum are there for your support; please use them. If you can’t make our office hours, let us know and we will schedule more. We want these projects to be rewarding and instructional, not frustrating and demoralizing. But, we don’t know when or how to help unless you ask.
+
+Discussion: Please be careful not to post spoilers.
+Installation
+
+If the following runs and you see the below window pop up where a line segment spins in a circle, you can skip to the pytorch instillation steps. You should use the conda environment for this since conda comes with the libraries we need.
+
+python autograder.py --check-dependencies
+
+Plot of a line
+
+For this project, you will need to install the following two libraries:
+
+    numpy, which provides support for fast, large multi-dimensional arrays.
+    matplotlib, a 2D plotting library.
+    pytorch, a library used for creating neural networks
+
+If you have a conda environment, you can install both packages on the command line by running:
+
+conda activate [your environment name]
+
+pip install numpy
+pip install matplotlib
+
+You will not be using these libraries directly, but they are required in order to run the provided code and autograder.
+
+If your setup is different, you can refer to numpy and matplotlib installation instructions. You can use either pip or conda to install the packages; pip works both inside and outside of conda environments.
+
+After installing, try the dependency check.
+Installing Pytorch
+
+Next you will need to install pytorch if you haven’t yet. First activate your conda environment:
+
+conda activate [your environment name]
+
+You can then follow the instructions here: Pytorch to download the latest version of Pytorch using either Conda or Pip. If you haven’t used Pytorch before, please use the CPU version. The CPU version of Pytorch is the least likely to cause any bugs or complications.
+Possible Numpy Bug: If you get any numpy errors, try downgrading to numpy version 1.24.3 or any version < 2.0.0.
+Pytorch Provided Functions (Part I)
+
+    tensor(): Tensors are the primary data structure in pytorch. They work very similarly to Numpy arrays in that you can add and multiply them. Anytime you use a pytorch function or feed an input into a neural network, you should try to make sure that your input is in the form of a tensor. You can change a python list to a tensor as such: tensor(data) where data is your n-dimentional list.
+    relu(input): The pytorch relu activation is called as such: relu(input). It takes in an input, and returns max(input, 0).
+    Linear: Use this class to implement a linear layer. A linear layer takes the dot product of a vector containing your weights, and the input. You must initialize this in your __init__ function like so: self.layer = Linear(length of input vector, length of output vector) and call it as such when running your model: self.layer(input). When you define a linear layer like this, Pytorch automatically creates weights and updates them during training.
+    movedim(input_vector, initial_dimension_position, final_dimension_position): This function takes in a matrix, and swaps the initial_dimension_position(passed in as an int), with final_dimension_position. This will be helpful in question 4.
+    cross_entropy(prediction, target): This function should be your loss function for any classification tasks(Questions 3-5). The further away your prediction is from the target, the higher a value this will return.
+    mse_loss(prediction, target): This function should be your loss function for any regression tasks(Question 2). It can be used in the same way as cross_entropy.
+
+All the data in the pytorch version will be provided to you in the form of a pytorch dataset object, which you will be transforming into a pytorch dataloader in order to help you easily create batch sizes.
+
+>>> data = DataLoader(training_dataset, batch_size = 64)
+>>> for batch in data:
+>>>   #Training code goes here
+
+For all of these questions, every batch returned by the DataLoader will be a dictionary in the form: {‘x’:features, ‘label’:label} with label being the value(s) we want to predict based off of the features.
+Question 1 (6 points):
+
+Before starting this part, be sure you have pytorch, numpy and matplotlib installed!
+
+In this part, you will implement a binary perceptron. Your task will be to complete the implementation of the PerceptronModel class in models.py.
+
+For the perceptron, the output labels will be either 1 or −1, meaning that data points (x, y) from the dataset will have y be a torch.Tensor that contains either 1 or −1 as its entries.
+
+Your tasks are to:
+
+    Fill out the init(self, dimensions) function. This should initialize the weight parameter in PerceptronModel. Note that here, you should make sure that your weight variable is saved as a Parameter() object of dimension 1 by dimensions. This is so that our autograder, as well as pytorch, recognize your weight as a parameter of your model.
+    Implement the run(self, x) method. This should compute the dot product of the stored weight vector and the given input, returning an Tensor object.
+    Implement get_prediction(self, x), which should return 1 if the dot product is non-negative or −1 otherwise.
+    Write the train(self) method. This should repeatedly loop over the data set and make updates on examples that are misclassified. When an entire pass over the data set is completed without making any mistakes, 100% training accuracy has been achieved, and training can terminate.
+    Luckily, Pytorch makes it easy to run operations on tensors. If you would like to update your weight by some tensor direction and a constant magnitude, you can do it as follows: self.w += direction * magnitude
+
+For this question, as well as all of the remaining ones, every batch returned by the DataLoader will be a dictionary in the form: {‘x’:features, ‘label’:label} with label being the value(s) we want to predict based off of the features.
+
+To test your implementation, run the autograder:
+
+python autograder.py -q q1
+
+Note: the autograder should take at most 20 seconds or so to run for a correct implementation. If the autograder is taking forever to run, your code probably has a bug.
+Neural Network Tips
+
+In the remaining parts of the project, you will implement the following models:
+
+    Q2: Non-linear Regression
+    Q3: Handwritten Digit Classification
+    Q4: Language Identification
+    Q5: Handwritten Digit Classification with CNNs
+
+Building Neural Nets
+
+Throughout the applications portion of the project, you’ll use Pytorch to create neural networks to solve a variety of machine learning problems. A simple neural network has linear layers, where each linear layer performs a linear operation (just like perceptron). Linear layers are separated by a non-linearity, which allows the network to approximate general functions. We’ll use the ReLU operation for our non-linearity, defined as relu(x)=max⁡(x,0)relu(x)=max(x,0). For example, a simple one hidden layer/ two linear layers neural network for mapping an input row vector xx to an output vector f(x)f(x) would be given by the function:
+f(x)=relu(x⋅W1+b1)⋅W2+b2
+f(x)=relu(x⋅W1​+b1​)⋅W2​+b2​
+
+where we have parameter matrices W1W1​ and W2W2​ and parameter vectors b1b1​ and b2b2​ to learn during gradient descent. W1W1​ will be an i×hi×h matrix, where ii is the dimension of our input vectors xx, and hh is the hidden layer size. b1b1​ will be a size hh vector. We are free to choose any value we want for the hidden size (we will just need to make sure the dimensions of the other matrices and vectors agree so that we can perform the operations). Using a larger hidden size will usually make the network more powerful (able to fit more training data), but can make the network harder to train (since it adds more parameters to all the matrices and vectors we need to learn), or can lead to overfitting on the training data.
+
+We can also create deeper networks by adding more layers, for example a three-linear-layer net:
+y^=f(x)=relu(relu(x⋅W1+b1)⋅W2+b2)⋅W3+b3
+y^​=f(x)=relu(relu(x⋅W1​+b1​)⋅W2​+b2​)⋅W3​+b3​
+
+Or, we can decompose the above and explicitly note the 2 hidden layers:
+h1=f1(x)=relu(x⋅W1+b1)
+h1​=f1​(x)=relu(x⋅W1​+b1​)
+h2=f2(h1)=relu(h1⋅W2+b2)
+h2​=f2​(h1​)=relu(h1​⋅W2​+b2​)
+y^=f3(h2)=h2⋅W3+b3
+y^​=f3​(h2​)=h2​⋅W3​+b3​
+
+Note that we don’t have a relurelu at the end because we want to be able to output negative numbers, and because the point of having relurelu in the first place is to have non-linear transformations, and having the output be an affine linear transformation of some non-linear intermediate can be very sensible.
+Batching
+
+For efficiency, you will be required to process whole batches of data at once rather than a single example at a time. This means that instead of a single input row vector xx with size ii, you will be presented with a batch of bb inputs represented as a b×ib×i matrix XX. We provide an example for linear regression to demonstrate how a linear layer can be implemented in the batched setting.
+Randomness
+
+The parameters of your neural network will be randomly initialized, and data in some tasks will be presented in shuffled order. Due to this randomness, it’s possible that you will still occasionally fail some tasks even with a strong architecture – this is the problem of local optima! This should happen very rarely, though – if when testing your code you fail the autograder twice in a row for a question, you should explore other architectures.
+Designing Architecture
+
+Designing neural nets can take some trial and error. Here are some tips to help you along the way:
+
+    Be systematic. Keep a log of every architecture you’ve tried, what the hyperparameters (layer sizes, learning rate, etc.) were, and what the resulting performance was. As you try more things, you can start seeing patterns about which parameters matter. If you find a bug in your code, be sure to cross out past results that are invalid due to the bug.
+    Start with a shallow network (just one hidden layer, i.e. one non-linearity). Deeper networks have exponentially more hyperparameter combinations, and getting even a single one wrong can ruin your performance. Use the small network to find a good learning rate and layer size; afterwards you can consider adding more layers of similar size.
+    If your learning rate is wrong, none of your other hyperparameter choices matter. You can take a state-of-the-art model from a research paper, and change the learning rate such that it performs no better than random. A learning rate too low will result in the model learning too slowly, and a learning rate too high may cause loss to diverge to infinity. Begin by trying different learning rates while looking at how the loss decreases over time.
+    Smaller batches require lower learning rates. When experimenting with different batch sizes, be aware that the best learning rate may be different depending on the batch size.
+    Refrain from making the network too wide (hidden layer sizes too large) If you keep making the network wider accuracy will gradually decline, and computation time will increase quadratically in the layer size – you’re likely to give up due to excessive slowness long before the accuracy falls too much. The full autograder for all parts of the project takes ~12 minutes to run with staff solutions; if your code is taking much longer you should check it for efficiency.
+    If your model is returning Infinity or NaN, your learning rate is probably too high for your current architecture.
+    Recommended values for your hyperparameters:
+        Hidden layer sizes: between 100 and 500.
+        Batch size: between 1 and 128. For Q2 and Q3, we require that total size of the dataset be evenly divisible by the batch size.
+        Learning rate: between 0.0001 and 0.01.
+        Number of hidden layers: between 1 and 3(It’s especially important that you start small here).
+
+Example: Linear Regression
+
+As an example of how the neural network framework works, let’s fit a line to a set of data points. We’ll start four points of training data constructed using the function y=7x0+8x1+3y=7x0​+8x1​+3. In batched form, our data is:
+X=[00011011]Y=[3111018]
+X=
+​0011​0101​
+​Y=
+​3111018​
+​
+
+Suppose the data is provided to us in the form of Tensors.
+
+>>> x
+torch.Tensor([[0,0],[0,1],[1,0],[1,1]])
+>>> y
+torch.Tensor([[3],[11],[10],[18]])
+
+Let’s construct and train a model of the form f(x)=x0⋅m0+x1⋅m1+bf(x)=x0​⋅m0​+x1​⋅m1​+b. If done correctly, we should be able to learn that m0=7m0​=7, m1=8m1​=8, and b=3b=3.
+
+First, we create our trainable parameters. In matrix form, these are:
+M=[m0m1]B=[b]
+M=[m0​m1​​]B=[b​]
+
+Which corresponds to the following code:
+
+m = Tensor(2, 1)
+b = Tensor(1, 1)
+
+A minor detail to remember is that tensors get initialized with all 0 values unless you initialize the tensor with data. Thus, printing them gives:
+
+>>> m
+torch.Tensor([[0],[0]])
+>>> b
+torch.Tensor([[0]])
+
+Next, we compute our model’s predictions for y. You must define a linear layer in your __init__() function as mentioned in the definition that is provided for Linear above.:
+
+predicted_y = self.Linear_Layer(x)
+
+Our goal is to have the predicted yy-values match the provided data. In linear regression we do this by minimizing the square loss:
+L=12N∑(x,y)(y−f(x))2
+L=2N1​(x,y)∑​(y−f(x))2
+
+We calculate our loss value:
+
+loss = mse_loss(predicted_y, y)
+
+Finally, after defining your neural network, In order to train your network, you will first need to initialize an optimizer. Pytorch has several built into it, but for this project use: optim.Adam(self.parameters(), lr=lr) where lr is your learning rate. Once you’ve defined your optimizer, you must do the following every iteration in order to update your weights:
+
+    Reset the gradients calculated by pytorch with optimizer.zero_grad()
+    Calculate your loss tensor by calling your get_loss() function
+    Calculate your gradients using loss.backward(), where loss is your loss tensor returned by get_loss
+    And finally, update your weights by calling optimizer.step()
+
+You can look at the official pytorch documentation for an example of how to use a pytorch optimizer().
+Question 2 (6 points): Non-linear Regression
+
+For this question, you will train a neural network to approximate sin⁡(x)sin(x) over [−2π,2π][−2π,2π].
+
+You will need to complete the implementation of the RegressionModel class in models.py. For this problem, a relatively simple architecture should suffice (see Neural Network Tips for architecture tips). Use mse_loss as your loss.
+
+Your tasks are to:
+
+    Implement RegressionModel.__init__ with any needed initialization.
+    Implement RegressionModel.run(RegressionModel.forward in pytorch) to return a batch_size by 1 node that represents your model’s prediction.
+    Implement RegressionModel.get_loss to return a loss for given inputs and target outputs. Note that get_loss takes in an input tensor, not a prediction, so you’ll have to pass x through your network before using a loss function.
+    Implement RegressionModel.train, which should train your model using gradient-based updates.
+
+There is only a single dataset split for this task (i.e., there is only training data and no validation data or test set). Your implementation will receive full points if it gets a loss of 0.02 or better, averaged across all examples in the dataset. You may use the training loss to determine when to stop training. Note that it should take the model a few minutes to train.
+
+python autograder.py -q q2
+
+Question 3 (6 points): Digit Classification
+
+For this question, you will train a network to classify handwritten digits from the MNIST dataset.
+
+Each digit is of size 28 by 28 pixels, the values of which are stored in a 784-dimensional vector of floating point numbers. Each output we provide is a 10-dimensional vector which has zeros in all positions, except for a one in the position corresponding to the correct class of the digit.
+
+Complete the implementation of the DigitClassificationModel class in models.py. The return value from DigitClassificationModel.run() should be a batch_size by 10 node containing scores, where higher scores indicate a higher probability of a digit belonging to a particular class (0-9). You should use cross_entropy as your loss. Do not put a ReLU activation in the last linear layer of the network.
+
+For both this question and Q4, in addition to training data, there is also validation data and a test set. You can use dataset.get_validation_accuracy() to compute validation accuracy for your model, which can be useful when deciding whether to stop training. The test set will be used by the autograder.
+
+To receive points for this question, your model should achieve an accuracy of at least 97% on the test set. For reference, our staff implementation consistently achieves an accuracy of 98% on the validation data after training for around 5 epochs. Note that the test grades you on test accuracy, while you only have access to validation accuracy – so if your validation accuracy meets the 97% threshold, you may still fail the test if your test accuracy does not meet the threshold. Therefore, it may help to set a slightly higher stopping threshold on validation accuracy, such as 97.5% or 98%.
+
+To test your implementation, run the autograder:
+
+python autograder.py -q q3
+
+Question 4 (7 points): Language Identification
+
+Language identification is the task of figuring out, given a piece of text, what language the text is written in. For example, your browser might be able to detect if you’ve visited a page in a foreign language and offer to translate it for you. Here is an example from Chrome (which uses a neural network to implement this feature):
+
+Screenshot of translate feature in browser
+
+In this project, we’re going to build a smaller neural network model that identifies language for one word at a time. Our dataset consists of words in five languages, such as the table below:
+Word 	Language
+discussed 	English
+eternidad 	Spanish
+itseänne 	Finnish
+paleis 	Dutch
+mieszkać 	Polish
+
+Different words consist of different numbers of letters, so our model needs to have an architecture that can handle variable-length inputs. Instead of a single input xx (like in the previous questions), we’ll have a separate input for each character in the word: x0,x1,⋯ ,xL−1x0​,x1​,⋯,xL−1​ where LL is the length of the word. We’ll start by applying a network finitialfinitial​ that is just like the networks in the previous problems. It accepts its input x0x0​ and computes an output vector h1h1​ of dimensionality dd:
+h1=finitial(x0)
+h1​=finitial​(x0​)
+
+Next, we’ll combine the output of the previous step with the next letter in the word, generating a vector summary of the the first two letters of the word. To do this, we’ll apply a sub-network that accepts a letter and outputs a hidden state, but now also depends on the previous hidden state h1h1​. We denote this sub-network as ff.
+h2=f(h1,x1)
+h2​=f(h1​,x1​)
+
+This pattern continues for all letters in the input word, where the hidden state at each step summarizes all the letters the network has processed thus far:
+h3=f(h2,x2)⋮
+h3​=f(h2​,x2​)⋮
+
+Throughout these computations, the function f(⋅,⋅)f(⋅,⋅) is the same piece of neural network and uses the same trainable parameters; finitialfinitial​ will also share some of the same parameters as f(⋅,⋅)f(⋅,⋅). In this way, the parameters used when processing words of different length are all shared. You can implement this using a for loop over the provided inputs xs, where each iteration of the loop computes either finitialfinitial​ or ff.
+
+The technique described above is called a Recurrent Neural Network (RNN). A schematic diagram of the RNN is shown below:
+
+RNN diagram
+
+Here, an RNN is used to encode the word “cat” into a fixed-size vector h3h3​.
+
+After the RNN has processed the full length of the input, it has encoded the arbitrary-length input word into a fixed-size vector hLhL​, where LL is the length of the word. This vector summary of the input word can now be fed through additional output transformation layers to generate classification scores for the word’s language identity.
+Batching
+
+Although the above equations are in terms of a single word, in practice you must use batches of words for efficiency. For simplicity, our code in the project ensures that all words within a single batch have the same length. In batched form, a hidden state hihi​ is replaced with the matrix HiHi​ of dimensionality batch_size by d.
+Design Tips
+
+The design of the recurrent function f(⋅,⋅)f(⋅,⋅) is the primary challenge for this task. Here are some tips:
+
+    Start with an architecture finitial(x)finitial​(x) of your choice similar to the previous questions, as long as it has at least one non-linearity.
+    You should use the following method of constructing f(⋅,⋅)f(⋅,⋅) given finitial(x)finitial​(x). The first transformation layer of finitialfinitial​ will begin by multiplying the vector x0x0​ by some weight matrix WxWx​ to produce z0=x0⋅Wxz0​=x0​⋅Wx​. For subsequent letters, you should replace this computation with zi=xi⋅Wx+hi⋅Whiddenzi​=xi​⋅Wx​+hi​⋅Whidden​ using an addition operation. In other words, you should replace a computation of the form z0 = self.Layer1(x, W) with a computation of the form self.Layer1(x) + self.Layer2(x).
+    If done correctly, the resulting function f(xi,hi)=g(zi)=g(zxi,hi)f(xi​,hi​)=g(zi​)=g(zxi​,hi​​) will be non-linear in both xx and hh.
+    The hidden size d should be sufficiently large.
+    Start with a shallow network for ff, and figure out good values for the hidden size and learning rate before you make the network deeper. If you start with a deep network right away you will have exponentially more hyperparameter combinations, and getting any single hyperparameter wrong can cause your performance to suffer dramatically.
+
+Your task
+
+Complete the implementation of the LanguageIDModel class.
+
+To receive full points on this problem, your architecture should be able to achieve an accuracy of at least 81% on the test set.
+
+To test your implementation, run the autograder:
+
+python autograder.py -q q4
+
+Disclaimer: This dataset was generated using automated text processing. It may contain errors. It has also not been filtered for profanity. However, our reference implementation can still correctly classify over 89% of the validation set despite the limitations of the data. Our reference implementation takes 10-20 epochs to train.
+Question 5 (4 points): Convolutional Neural Networks
+
+Oftentimes when training a neural network, it becomes necessary to use layers more advanced than the simple Linear layers that you’ve been using. One common type of layer is a Convolutional Layer. Convolutional layers make it easier to take spatial information into account when training on multi-dimentional inputs. For example, consider the following Input:
+Input=[x11x12x13…x1nx21x22x23…x2n⋮⋮⋮⋱⋮xd1xd2xd3…xdn]
+Input=
+​x11​x21​⋮xd1​​x12​x22​⋮xd2​​x13​x23​⋮xd3​​……⋱…​x1n​x2n​⋮xdn​​
+​
+
+If we were to use a linear layer, similar to what was done in Question 2, in order to feed this input into your neural network you would have to flatten it into the following form:
+Input=[x11x12x13…x1n…xdn]
+Input=[x11​​x12​​x13​​…​x1n​…xdn​​]
+
+But in some problems, such as image classification, it’s a lot easier to recognize what an image is if you are looking at the original 2-dimentional form. This is where Convolutional layers come in to play.
+
+Rather than having a weight be a 1-dimentional vector, a 2d Convolutional layer would store a weight as a 2d matrix:
+Weights=[w11w12w21w22]
+Weights=[w11​w21​​w12​w22​​]
+
+And when given some input, the layer then convolves the input matrix with the output matrix. After doing this, a Convolutional Neural Network can then make the output of a convolutional layer 1-dimensional and passes it through linear layers before returning the final output.
+
+A 2d convolution can be defined as follows:
+Output=[a11a12a13…a1na21a22a23…a2n⋮⋮⋮⋱⋮ad1ad2ad3…adn]
+Output=
+​a11​a21​⋮ad1​​a12​a22​⋮ad2​​a13​a23​⋮ad3​​……⋱…​a1n​a2n​⋮adn​​
+​
+
+Where aijaij​ is created by performing an element wise multiplication of the Weights matrix and the section of the input matrix that begins at xijxij​ and has the same width and height as the Weights matrix. We then take the sum of the resulting matrix to calculate aijaij​. For example, if we wanted to find a22a22​, we would multiply Weights by the following matrix:
+[x22x23x32x33]
+[x22​x32​​x23​x33​​]
+
+to get
+[x22∗w11x23∗w12x32∗w21x33∗w22]
+[x22​∗w11​x32​∗w21​​x23​∗w12​x33​∗w22​​]
+
+before taking the sum of this matrix a22=x22∗w11+x23∗w12+x32∗w21+x33∗w22a22​=x22​∗w11​+x23​∗w12​+x32​∗w21​+x33​∗w22​
+
+Sometimes when applying a convolution, the Input matrix is padded with 00’s to ensure that the output and input matrix can be the same size. However, in this question that is not required. As a result, your output matrix should be smaller than your input matrix.
+
+Your task is to first fill out the Convolve function in models.py. This function takes in an input matrix and weight matrix, and Convolves the two. Note that it is guaranteed that the input matrix will always be larger than the weights matrix and will always be passed in one at a time, so you do not have to ensure your function can convolve multiple inputs at the same time.
+
+After doing this, complete the DigitConvolutionalModel() class in models.py. You can reuse much of your code from question 3 here.
+
+The autograder will first check your convolve function to ensure that it correctly calculates the convolution of two matrices. It will then test your model to see if it can achieve and accuracy of 80\% on a greatly simplified subset MNIST dataset. Since this question is mainly concerned with the Convolve() function that you will be writing, your model should train relatively quick.
+
+In this question, your Convolutional Network will likely run a bit slowly, this is to be expected since packages like Pytorch have optimizations that they use to speed up convolutions. However, this should not affect your final score since we provide you with an easier version of the MNIST dataset to train on.
+
+Model Hints: We have already implemented the convolutional layer and flattened it for you. You can now treat the flattened matrix as you would a regular 1-dimensional input by passing it through linear layers. You should only need a couple of small layers in order to achieve an accuracy of 80%.
+EXTRA CREDIT Question 6 (1 point): Attention
+
+Attention is a relatively new concept in machine learning used in tasks such as natural language processing to, in theory, help neural networks learn to prioritize the important parts of a piece of text. In this question, we will implement a type of attention mechanism called “Scaled Dot-Product Attention” which is given by the following formula.
+softmax(M((K)(Q)Tdk))(V)
+softmax(M(d
+​k​(K)(Q)T​))(V)
+
+Here, Q,K,VQ,K,V are all input matrices, not necessarily vectors. In our case, we will let QQ, KK, and VV all be equal to the input matrix with a linear layer applied to them, for reasons that will be seen in the next question. The pytorch function matmul will have to be used to multiply them. Additionally, dkdk​ is the layer size that’s being used. Finally, the notation ATAT is used to indicate taking the transpose of a matrix. This can be done by switching the 2nd and 3rd dimentions of a matrix using movedim as was done in question 4.
+
+Additionally, you will be applying a mask(represented by MM) to your attention layer. A mask is a boolean matrix which controls which elements in our input that our neural network can look at. In this case we want to implement a causal mask, which prevents the neural network from “looking ahead” to future terms when processing a sequence of characters. More information on how to do this can be found in the skeleton code.
+
+The only task to receive credit for question 6 is to fill out the AttentionBlock class in models.py. Question 7, which is optional, will walk you through using it in generative neural network
+Question 7 (0 points but fun): character-GPT
+
+This question a modified version of Andrej Karpathy’s “minGPT”, which is a good resource if you’d like to play with a more complex, and more powerful, version of the model you will create in this question
+
+Now that you’ve built an attention block, you can now move on to a type of architecture which has become extremely popular over the past few years: Transformers. Some of the more widely used transformer based models, including GPT, are focused on text generation.
+
+You will be building a much smaller version of a generative model that will be trained on a large sample of Shakespeare’s plays. To make it possible for this to be trained on a CPU, this model will generate the next character in a sequence rather than the next word. All of the code you will need to edit is located in gpt_model.py
+
+The general structure of GPT looks like the following:
+
+Model of transformer source: Improving Language Understanding by Generative Pre-Training
+
+The blue rectangle in the center represents a single transformer block. It consists of the following steps:
+
+    Call your attention block
+    Sum the output of step 1 and the input of the transformer block
+    Normalize the output of step 2
+    Apply a single linear layer followed by a relu activation
+    Add the output of 3 and 4
+    Normalize the output of 5
+
+You should implement this in the forward function of the Transformer_Block class. All the layers you need have already been initialized in __init__
+
+The second part of this question is to assemble the entire architecture in the forward function of GPT. Using the image above as a rough guide, the first step is to create an “embedding layer”. For the sake of this project, you can consider it to be essentially a linear layer, this, along with all the other layers have already been initialized for you in the __init__() function. Next, you will have to apply a series of transformer blocks on the input. While the image above uses 12, since you will be working with a smaller dataset, you won’t need nearly as much. Finally, you will apply a normalization layer, then one last linear layer before returning the output.
+
+Note that this last layer should not have an activation function. The last layer returns the likelihood of the next character being any specific letter so it should behave similarly to the final layers of the previous classification problems you worked on.
+
+To train this model, run the command:
+
+python chargpt.py
+
+Next steps: Since this question will not be graded, you are encouraged to try increasing the size of the network, replace input.txt with a different text file you’d like to work on, or investigate using a larger block size, more information on the variables you can change can be found in the file chargpt.py. You can also take a look at “minGPT” for a more powerful version you can play around with.
+Submission
+
+In order to submit your project upload the Python files you edited. For instance, use Gradescope’s upload on all .py files in the project folder.
+
+The full project autograder takes ~12 minutes to run for the staff reference solutions to the project. If your code takes significantly longer, consider checking your implementations for efficiency.
+
+Please specify any partner you may have worked with and verify that both you and your partner are associated with the submission after submitting.
+