Insight UBC: Deliverable 2

Goals and specs

1. Add and remove a dataset containing information about UBC classrooms

Goals and specs

1. Add and remove a dataset containing information about UBC classrooms

rooms.zip

addDataset(...)

Parse it

Keep data structure in a variable

Save it to disk

Caching

same workflow as courses.zip BUT with different file types

 

InsightDatasetKind.Rooms

(kind)

Goals and specs

2. Answer advanced queries about UBC courses and rooms.

Goals and specs

Examples:

"what's the average of CPSC 340 ?"

2. Answer advanced queries about UBC courses and rooms.

Goals and specs

2. Answer advanced queries about UBC courses. (current solution)

{
  "WHERE":{
    "AND": [
      { "IS":{ "courses_id": "*340*" } },
      { "IS":{ "courses_dept": "*cpsc*" }}
    ]
   },
   "OPTIONS":{
      "COLUMNS":[
         "courses_id", "courses_avg"
      ],
      "ORDER":"courses_avg"
   }
}

Goals and specs

{"result": [
  { "courses_id": "340", 
    "courses_avg": 68.4 },
  { "courses_id": "340", 
    "courses_avg": 68.4 },
  { "courses_id": "340", 
    "courses_avg": 72.65 },
  { "courses_id": "340", 
    "courses_avg": 72.65 },
  { "courses_id": "340", 
    "courses_avg": 72.94 },
  { "courses_id": "340", 
    "courses_avg": 72.94 },
  ...
]}
{
  "WHERE":{
    "AND": [
      { "IS":{ "courses_id": "*340*" } },
      { "IS":{ "courses_dept": "*cpsc*" }}
    ]
   },
   "OPTIONS":{
      "COLUMNS":[
         "courses_id", "courses_avg"
      ],
      "ORDER":"courses_avg"
   }
}

not what we
were looking for

2. Answer advanced queries about UBC courses. (current solution)

Goals and specs

2. Answer advanced queries about UBC courses. (new query engine)

{
  "result": [
    {
      "courses_id": "340",
      "overallAvg": 75.69
    }
  ]
}
{
  "WHERE":{
    "AND": [
      { "IS":{ "courses_id": "*340*" } },
      { "IS":{ "courses_dept": "*cpsc*" }}
    ]
   },
   "OPTIONS":{
      "COLUMNS":[
         "courses_id", "overallAvg"
      ]
   },
   "TRANSFORMATIONS":{
      "GROUP":[ "courses_id"],
      "APPLY": [
      	{
      	  "overallAvg": {
      	    "AVG": "courses_avg"
      	  }
      	}
      ]
   }
}

this is the right result!

Goals and specs

2. New EBNF:

QUERY ::='{'BODY ', ' OPTIONS (', ' TRANSFORMATIONS)? '}'

BODY ::= 'WHERE:{' (FILTER)? '}'
OPTIONS ::= 'OPTIONS:{' COLUMNS ', ' (SORT)?'}'
TRANSFORMATIONS ::= 'TRANSFORMATIONS: {' GROUP ', ' APPLY '}'

FILTER ::= (LOGICCOMPARISON | MCOMPARISON | SCOMPARISON | NEGATION)

LOGICCOMPARISON ::= LOGIC ':[{' FILTER ('}, {' FILTER )* '}]'  
MCOMPARISON ::= MCOMPARATOR ':{' key ':' number '}'  
SCOMPARISON ::= 'IS:{' key ':' [*]? inputstring [*]? '}'  // inputstring may have option * characters as wildcards
NEGATION ::= 'NOT :{' FILTER '}'

LOGIC ::= 'AND' | 'OR'
MCOMPARATOR ::= 'LT' | 'GT' | 'EQ'

COLUMNS ::= 'COLUMNS:[' ((key|applykey) ',')* (key|applykey) ']'
SORT ::= 'ORDER: ' ('{ dir:'  DIRECTION ', keys: [ ' ORDERKEY (',' ORDERKEY)* ']}') | ORDERKEY
DIRECTION ::= 'UP' | 'DOWN'  
ORDERKEY ::= key | applykey

GROUP ::= 'GROUP: [' (key ',')* key ']'                                                          
APPLY ::= 'APPLY: [' (APPLYRULE (', ' APPLYRULE )* )? ']'  
APPLYRULE ::= '{' applykey ': {' APPLYTOKEN ':' key '}}'
APPLYTOKEN ::= 'MAX' | 'MIN' | 'AVG' | 'COUNT' | 'SUM'                           

key ::= string '_' string
inputstring ::= [^*]* // zero or more of any character except asterisk.
applykey ::= [^_]+ // one or more of any character except underscore.

Activity #1

Take a few minutes and discuss with your partner what strategies could be used to integrate the new data set into your existing solution. 

 

Share your strategy (not your code) with other teams.

  - Are they equal? 

  - How they differ?

  - Which one better suits our problem? 

  - Wait. Do we have a problem? What is our problem?

~5 min

addDataset(...)

new code

kind?

existing code

parse courses

parse rooms

Keep data structure in a variable

Save it to disk

Activity #1. Do we have a problem?


return new Promise(function (fulfill, reject) {
    try {
        const myZip = new Zip();
        if (kind === InsightDatasetKind.Courses) {
            // Extract the content
            zip = myZip.extract();
            for (file in files inside zip) {
                try{
                    contents = file.getContent();
                    for (section in results) {
                        // do something
                    }
                } catch (err) {
                    // do something
                }
            });
            // more code here
        } else if (kind === InsightDatasetKind.Rooms) {
            // We are just going to copy and paste the code above and generate ourselves a problem
        } else {
            // keep doing the same strategy until our code is doomed
        }
        // more code here
        // even more code
        // ...
    } catch (err) {
        // do something
    }
});

Do we have a problem?


return new Promise(function (fulfill, reject) {
    try {
        const myZip = new Zip();
        if (kind === InsightDatasetKind.Courses) {
            // Extract the content
            zip = myZip.extract();
            for (file in files inside zip) {
                try{
                    contents = file.getContent();
                    for (section in results) {
                        // do something
                    }
                } catch (err) {
                    // do something
                }
            });
            // more code here
        } else if (kind === InsightDatasetKind.Rooms) {
            // Extract the content
            zip = myZip.extract();
            for (file in files inside zip) {
                try{
                    contents = file.getContent();
                    for (section in results) {
                        // do something
                    }
                } catch (err) {
                    // do something
                }
            });
 
        } else {
            // keep doing the same strategy until our code is doomed
        }
        // more code here
        // even more code
        // ...
    } catch (err) {
        // do something
    }
});

Do we have a problem?

1. Code Readability

2. Code Maintainability

3. Code Testability


return new Promise(function (fulfill, reject) {
    try {
        const myZip = new Zip();
        if (kind === InsightDatasetKind.Courses) {
            // Extract the content
            zip = myZip.extract();
            for (file in files inside zip) {
                try{
                    contents = file.getContent();
                    for (section in results) {
                        // do something
                    }
                } catch (err) {
                    // do something
                }
            });
            // more code here
        } else if (kind === InsightDatasetKind.Rooms) {
            // Extract the content
            zip = myZip.extract();
            for (file in files inside zip) {
                try{
                    contents = file.getContent();
                    for (section in results) {
                        // do something
                    }
                } catch (err) {
                    // do something
                }
            });
        else if (kind === InsightDatasetKind.X) {
            // Extract the content
            zip = myZip.extract();
            for (file in files inside zip) {
                try{
                    contents = file.getContent();
                    for (section in results) {
                        // do something
                    }
                } catch (err) {
                    // do something
                }
            });
            // more code here
        } else if (kind === InsightDatasetKind.Y) {
            // Extract the content
            zip = myZip.extract();
            for (file in files inside zip) {
                try{
                    contents = file.getContent();
                    for (section in results) {
                        // do something
                    }
                } catch (err) {
                    // do something
                }
            });
        } else if (kind === InsightDatasetKind.Z) {
            // Extract the content
            zip = myZip.extract();
            for (file in files inside zip) {
                try{
                    contents = file.getContent();
                    for (section in results) {
                        // do something
                    }
                } catch (err) {
                    // do something
                }
            });
        } else {
            // keep doing the same strategy until our code is doomed
        }
        // more code here
        // even more code
        // ...
    } catch (err) {
        // do something
    }
});

Do we have a problem?

1. Code Readability

2. Code Maintainability

3. Code Testability

Activity #2

Clone the repo at 

https://github.ugrad.cs.ubc.ca/CPSC310-2018W-T1/tutorial_abstractions.git

 

Extend the functionality of the addNumberToProcess method in the SimpleMathMachine class to allow the input of words for the numbers 0-99 (e.g. input 'sixty-three' for 63).

 

For simplicity, assume only valid inputs are passed.

~10-15 min

private parseWord(input: string) : number {
    let a = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve',     
    'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'];
    let b = ['twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'];
    let idx = a.indexOf(input);
        
    if (idx > -1) {
        return idx
    }
    else {
        let words = input.split('-');
        let num1 = b.indexOf(words[0]) + 2;
        let num2 = a.indexOf(words[1]);

        return(num1 * 10 + num2);
    }
}

Refactoring

Is the process of restructuring existing code without changing its behavior -- Wikipedia

Extract method: roughly moves some code to a smaller and well contained method

Activity #3

Now that we have a glimpse about how to encapsulate some of the dataset behaviour into methods, try and think of at least 3 methods you could extract within your addDataset method with your partner.

 

  - Does this help you integrating the new rooms dataset?

  - How can you be sure that your code still works after refactoring?

~5 min

Activity #3

 

  - Does this help you integrating the new rooms dataset?

Your code should be open for extension, but closed for modification [more to come]

Test suites! Never change your code if you don't have tests that cover that functionality

  - How can you be sure that your code still works after refactoring?

D2 - Further clarifications

The rooms dataset

index.htm
/campus/
/campus/discover/
/campus/discover/buildings-and-classrooms/
/campus/discover/buildings-and-classrooms/AAC
/campus/discover/buildings-and-classrooms/ACEN
/campus/discover/buildings-and-classrooms/ACU
/campus/discover/buildings-and-classrooms/AERL
/campus/discover/buildings-and-classrooms/ALRD
...

- Files in HTML format, parsable using the parse5 package

- Each file other than index.htm represents a building and its rooms

- All buildings linked from the index should be considered valid buildings

zip.files

?

?

single file
e.g. AAC

your datastructure

parse5 datastructure

parse5

const document = parse5.parse(<my file>) as parse5.AST.Default.Document;*

* code snippet taken from http://inikulin.github.io/parse5/
* sample html/output taken from http://astexplorer.net/#/1CHlCXc4n4 

<html>
<body>
<h1>
<p>
My first heading
My first paragraph

- Inspect the html files in the dataset, take a look at them and try to find useful information

tree node for that element

current html element that I'm inspecting

HTTP

GET http://skaha.cs.ubc.ca:11316/api/v1/team666/6245%20Agronomy%20Road%20V6T%201Z4
6245 Agronomy Road V6T 1Z4
6245%20Agronomy%20Road%20V6T%201Z4

address

URL-encoded address

status: 200,
body: {"lat":49.26125,"lon":-123.24807}

request

response

* Take a look at https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol for a more comprehensive explanation of HTTP

client

server

const request = require('request');

request({ 
     method: 'GET', 
     uri: 'http://cs310.ugrad.cs.ubc.ca:11316/api/v1/
project_<CSID1>_<CSID2>/6245%20Agronomy%20Road%20V6T%201Z4',
     gzip: true
}).on('response', function(response) {
     console.log("status: " + response.statusCode);
}).on('data', function(data) {
     // decompressed data as it is received
     console.log('decoded chunk: ' + data)
});
status: 200
decoded chunk: {
    "lat":49.26125,
    "lon":-123.24807
}

* https://github.com/request/request

* This code will not work in your project ! You are supposed to use the http package instead of request.

HTTP

const request = require('request');

request({ 
     method: 'GET', 
     uri: 'http://cs310.ugrad.cs.ubc.ca:11316/api/v1/???',
     gzip: true
}).on('data', function(data) {
     // decompressed data as it is received
     console.log('decoded chunk: ' + data)
}).on('response', function(response) {
     console.log("status: " + response.statusCode);
});
status: 404
decoded chunk: {
    "code":"ResourceNotFound",
    "message":"/api/v1/ does not exist"
}

HTTP

* https://github.com/request/request

* This code will not work in your project ! You are supposed to use the http package instead of request.

Activity #4

Extend the functionality of the addNumberToProcess method in SimpleMathMachine again, this time to allow the input of expressions (e.g. '(3+sqrt(2))')

 

 - Doing this all ourselves would be a lot of work; fortunately someone else has done the hard work for us already

- Use the math.js RESTful api (details at http://api.mathjs.org/) to compute the results of the expressions

~10-15 min


request({ 
     method: 'GET', 
     uri: <fill in with api request here>,
     gzip: true,
     encoding: 'utf8' // This specifies that the
                      // returned data should be a string

}).on('data', function(data) {
     // decompressed data as it is received
     console.log('decoded chunk: ' + data)
}).on('response', function(response) {
     console.log("status: " + response.statusCode);
});

HTTP

* https://github.com/request/request

* This code will not work in your project ! You are supposed to use the http package instead of request.

Deliverable 2 tutorial

By csatterfield

Source: Arthur Marques

Deliverable 2 tutorial

  • 209

More from csatterfield