Inspiration to write Go?
docker
NSQ
Operating system level virtualization
Realtime message processing system designed
I've worked at Google since 2004. I've worked on the Go programming language team since 2012. You can find my posts and talks on Go at sameer.io/go.
Sameer Ajmani
Outline
What in the world is Go?
Who uses it anyway?
Some basic Go to get you started
What were the Problems faced at ?
How did they solve their problems with Go?
What is Go?
Go is an open source programming language that makes it easy to build simple, reliable, and efficient software.
Why did Go come up in the first place?
Why did Google design a new language?
SCALE.
The two major problems Google was facing was scaling in Infrastructure and scaling in Engineering.
Infrastructure
Everyday services and jobs run on thousands of machines
Thousands of rpc interaction between services
lots of going on at once
Engineering
5000+ Developers
Lots of changes to the codebase
50 million test cases executing per day
single code tree
Skip to content
Search…
All gists
GitHub
New gist
@minhajuddinkhan
Star 0
Fork 0@ptitfredptitfred/server.cpp
Created 8 years ago • Report abuse
Embed
<script src="https://gist.github.com/ptitfred/612357.js"></script>
Download ZIP
Code Revisions 3
C++ HTTP server
Raw
server.cpp
#include "server.hpp"
#include <iostream>
#include <fstream>
#include <sstream>
using std::cout;
using std::cerr;
using std::cin;
using std::endl;
using std::fstream;
using std::ios;
using std::string;
using std::stringstream;
#include <boost/filesystem.hpp>
namespace fs = boost::filesystem;
#define PREFIX "[SERV] "
namespace bbl {
int Server::_static_handle( void * cls, // Server*
struct MHD_Connection * connection,
const char * url,
const char * method,
const char * version,
const char * upload_data,
size_t * upload_data_size,
void ** ptr ) {
Server* server = (Server*) cls;
return server->_handle(connection, url, method, version, upload_data, upload_data_size, ptr);
}
int Server::_handle( struct MHD_Connection * connection,
const char * url,
const char * method,
const char * version,
const char * upload_data,
size_t * upload_data_size,
void ** ptr ) {
static int dummy;
// PUT and POST data must first be gathered
// But not yet implemented => 404
if (strcmp("PUT", method) == 0 || strcmp("POST", method) == 0) {
// Not yet handled => 404
cerr << PREFIX << method << " not yet handled..." << endl;
return MHD_NO;
}
if (&dummy != *ptr) {
/* The first time only the headers are valid,
do not respond in the first round... */
*ptr = &dummy;
return MHD_YES;
}
*ptr = NULL; /* clear context pointer */
result r = this->_handler->handle(method, url);
if (r.httpCode == MHD_HTTP_OK) {
struct MHD_Response * response;
int ret;
response = MHD_create_response_from_data( r.data.size(),
(void*) r.data.data(),
MHD_NO,
MHD_YES); // Ask MHD to copy data
MHD_add_response_header( response, "Content-Type", r.mimeType != "" ? r.mimeType.c_str() : "text/html");
if (r.encoding != "")
MHD_add_response_header( response, "Content-Encoding", r.encoding.c_str());
ret = MHD_queue_response(connection,
MHD_HTTP_OK, // 202
response);
MHD_destroy_response(response);
return ret;
} else {
return MHD_queue_response(connection, r.httpCode, NULL);
}
}
int Server::_static_filter(
void *cls, // Server*
const struct sockaddr * addr,
socklen_t addrlen) {
Server* server = (Server*) cls;
return server->_filter(addr, addrlen) ? MHD_YES : MHD_NO;
}
bool Server::_filter(const struct sockaddr * addr, socklen_t addrlen) {
// TODO filter non local addresses.
return true;
}
bool Server::start() {
cout << PREFIX << "Starting..." << endl;
this->_daemon = MHD_start_daemon(this->_threadPoolSize >0 ? MHD_USE_SELECT_INTERNALLY : MHD_USE_THREAD_PER_CONNECTION,
this->_port,
& (Server::_static_filter), this,
& (Server::_static_handle), this,
MHD_OPTION_THREAD_POOL_SIZE, this->_threadPoolSize,
MHD_OPTION_END);
if (this->_daemon == NULL)
return false;
cout << PREFIX << "port: " << this->_port << endl;
cout << PREFIX << "threads: " << this->_threadPoolSize << endl;
return true;
}
void Server::stop() {
MHD_stop_daemon(this->_daemon);
}
} // namespace ::bbl
class DefaultHandler : public bbl::Handler {
public:
DefaultHandler(const std::string& chroot) : _chroot(chroot), prefix("[DH ] ") {}
~DefaultHandler() {}
public:
bbl::result handle(const string& method, const string& url) {
cout << prefix << method << ":" << url << endl;
bbl::result r;
fs::path pPath( _chroot / url);
if ( !fs::exists( pPath ) ) {
cout << prefix << " Not found " << pPath.file_string() << endl;
r.httpCode = MHD_HTTP_NOT_FOUND;
return r;
}
if ( fs::is_directory( pPath ) && fs::exists( pPath / "index.html") ) {
pPath = pPath / "index.html";
}
const string sPath = pPath.file_string();
if ( fs::is_directory ( pPath ) ) {
// list files
cout << prefix << " Listing directory" << endl;
fs::directory_iterator end_iter;
stringstream buffer;
buffer << "<html><head><title>" << pPath.file_string() << "</title></head>" << endl;
buffer << "<body>" << endl;
buffer << "<h3>" << pPath.file_string() << "</h3>" << endl;
buffer << "</p>" << endl;
for ( fs::directory_iterator dir_itr( pPath );
dir_itr != end_iter;
++dir_itr ) {
string link = dir_itr->path().filename();
try {
buffer << "<a href='" << link << "'>" << link << "</a><br/>" << endl;
} catch ( const std::exception & ex ) {
cout << link << " " << ex.what() << endl;
}
}
buffer << "</p>" << endl;
buffer << "</body>" << endl;
buffer << "</html>" << endl;
r.httpCode = MHD_HTTP_OK;
r.mimeType = MIME_HTML;
r.data = buffer.str();
} else {
fstream file(sPath.c_str(), ios::binary | ios::ate | ios::in);
int size = file.tellg();
cout << prefix << " Serving " << sPath << " (" << size << " bytes)" << endl;
char* memblock = new char [size];
file.seekg (0, std::ios::beg);
file.read (memblock, size);
file.close();
r.httpCode = MHD_HTTP_OK;
r.data = string(memblock, size);
}
return r;
};
private:
string prefix;
fs::path _chroot;
};
int main(int argc, char** argv) {
if (argc != 4) {
cerr << "Usage : server port threadPoolSize webroot" << endl;
return 1;
}
bbl::Server s(atoi(argv[1]), atoi(argv[2]), new DefaultHandler(argv[3]));
s.start();
char c;
cin >> c;
s.stop();
}
Raw
server.hpp
#include <sys/types.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <stdarg.h>
#include <stdint.h>
#include <microhttpd.h>
#include <string>
namespace bbl {
struct result {
int httpCode;
std::string data;
std::string mimeType;
std::string encoding;
};
const std::string MIME_HTML = "text/html";
class Handler {
public:
virtual result handle(const std::string& method, const std::string& url) =0;
};
class Server {
public:
Server(int port, int threadPoolSize, Handler* handler) : _handler(handler), _port(port), _threadPoolSize(threadPoolSize) {}
~Server() { stop(); }
public:
bool start();
void stop();
private:
bool _filter( const struct sockaddr * addr,
socklen_t addrlen);
static int _static_filter(
void *cls, // Server*
const struct sockaddr * addr,
socklen_t addrlen);
/**
* <p>Barely delegates to _handler->handle(...).</p>
*/
int _handle(
struct MHD_Connection * connection,
const char * url,
const char * method,
const char * version,
const char * upload_data,
size_t * upload_data_size,
void ** ptr
);
/**
* <p>First parameter is a Server* ; other args are delegated to cls->_handle(...).</p>
*/
static int _static_handle(
void * cls, // Server*
struct MHD_Connection * connection,
const char * url,
const char * method,
const char * version,
const char * upload_data,
size_t * upload_data_size,
void ** ptr
);
private:
Handler* _handler;
int _port;
int _threadPoolSize;
struct MHD_Daemon * _daemon;
};
} // namespace bbl;
https://gist.github.com/ptitfred/612357
Large codebase and many services to manage?
Its not just google's problem, right?
Who uses Go?
Go claims to be readable and talks a lot about simplicity
Java
Classes
Access Modifiers
Polymorphism
Dynamic Binding
Inheritance
Method Overloading
Method Overriding
Javascript
Promises
Context
'this' (what a pain)
Arrow functions
async/await
Destructuring
Hoisting
So, lets compare Go with the two most popular languages in their paradigm, OOP and functional, Java and JavaScript.
Both languages have a certain set of stuff that makes the language full of features, providing their own signature.
Simplicity
Go's approach to make stuff simple was straight forward. Pull out the extra stuff.
Taking Stuff Out
No Classes
No Inheritance
No Constructors
No Finals
No Exceptions
No User defined generics
No Futures/Promises
No Callbacks
Why
Though?
Clarity.
Reading code should be fairly simple and straight forward.
GoLang's theory about code is giving the code reader more flexibility over writing code
The most frequent iterations over a code base is not writing new features. Its more about maintaining and enhancing already implemented codebases.
Enter Code.
Lets see what its all about.
package main
import "fmt"
func main() {
fmt.Println("Hello, 世界")
}
package main
import(
"fmt"
"log"
"net/http"
)
func main() {
http.HandleFunc("/hello", helloHandler)
fmt.Println("serving http server on localhost:3000")
log.Fatal(http.ListenAndServe(":3000", nil))
}
func helloHandler(w http.ResponseWriter, r *http.Request) {
log.Println("serving", r.URL)
fmt.Fprintf(w, "Hello, world!")
}
Types follow name in declarations.
Exported names are Capitalized, Unexported arent
package main
import(
google "github.com/minhajuddinkhan/gocon/google"
"time"
"fmt"
)
func main() {
start := time.Now()
results, err := google.Search("golang")
elapsed := time.Since(start)
fmt.Println(results)
fmt.Println(elapsed, err)
}
What about futures?
Go works with CSP Model
Easy to understand.
Go Primitives:
Goroutines Channels Select statement
Communicating Single Processes
Concurrent programs are structured independent processes that execute sequentially and communicate by passing messages.
1. Goroutines
-
Light weight threads
-
Go programs can have thousands of them
-
Go runtime scheduler maps them on OS threads
-
Blocked goroutines dont use an OS thread
go f(args)
2. Channels
-
Provides communication between goroutines
-
Think of them as type synchronized queues.
c := make chan(string)
c <- "hello"
s := <- c
fmt.Println(s) //Prints "hello"
3. Select
-
Select statement blocks until communication can proceed.
-
only the selected case runs.
select {
case msg1 := <-c1:
fmt.Println("received", msg1)
case msg2 := <-c2:
fmt.Println("received", msg2)
}
Given a query, returns a set of result consisting of urls, images, and videos. (ads too)
Google sends this query to WebSearch, ImageSearch, Youtube, Maps, News.. etc and then mix the results.
< The What? />
< The How? />
Implementation?
A fake search framework
var (
Web = FakeSearch("web", "The Go Programming Language", "http://golang.org")
Image = FakeSearch("image", "The Go gopher", "https://blog.golang.org/gopher/gopher.png")
Video = FakeSearch("video", "Errors are values", "https://www.youtube.com/watch?v=cN_DpYBzKso")
)
type SearchFunc func(query string) Result // HL
func FakeSearch(kind, title, url string) SearchFunc {
return func(query string) Result {
duration := time.Duration(rand.Intn(100)) * time.Millisecond
time.Sleep(duration) // HL
return Result{
Title: fmt.Sprintf("%s(%q): %s", kind, query, title),
URL: url,
}
}
}
package main
import(
google "github.com/minhajuddinkhan/gocon/google"
"time"
"fmt"
)
func main() {
start := time.Now()
results, err := google.Search("golang")
elapsed := time.Since(start)
fmt.Println(results)
fmt.Println(elapsed, err)
}
package google
type Result struct {
Title, URL string
}
func Search(query string) ([]Result, error) {
results := []Result{
Web(query),
Image(query),
Video(query),
}
return results, nil
}
Sequential
google.Search("golang")
But this is Go.
We can do better
We do stuff concurrently here.
package google
func SearchParallel(query string) ([]Result, error) {
c := make(chan Result)
go func() { c <- Web(query) }()
go func() { c <- Image(query) }()
go func() { c <- Video(query) }()
return []Result{<-c, <-c, <-c}, nil
}
Parallel
google.SearchParallel("golang")
Run the web, image and video query in parallel.
But we can do better.
package google
import (
"errors"
"time"
)
func SearchTimeout(query string, timeout time.Duration) ([]Result, error) {
timer := time.After(timeout) // HL
c := make(chan Result, 3)
go func() { c <- Web(query) }()
go func() { c <- Image(query) }()
go func() { c <- Video(query) }()
var results []Result
for i := 0; i < 3; i++ {
select { // HL
case result := <-c: // HL
results = append(results, result)
case <-timer: // HL
return results, errors.New("timed out")
}
}
return results, nil
}
Leave
Out
Slow Servers
We can still do better
func First(replicas ...SearchFunc) SearchFunc { // HL
return func(query string) Result {
c := make(chan Result, len(replicas))
searchReplica := func(i int) {
c <- replicas[i](query)
}
for i := range replicas {
go searchReplica(i) // HL
}
return <-c
}
}
Replicate our Web, Image and Video Repositories
Ask all of them, pick the first and the fastest response.
go func() { c <- Web(query) }()
go func() { c <- Image(query) }()
go func() { c <- Video(query) }()
All of the concurrencuy and futures is encapsulated within these functions.
this is one of the most critical powers of Go.
From
Slow
Sequential
Failure sensitive
Futuristic looking code
To
Fast
Concurrent
Replicated
Robust code
No locks. No condition variables. No Futures. No Callbacks
Hopefully I've got you guys want to learn Go.
Thankyou.
Google's
By Minhaj Khan
Google's
- 404