Beyond unit tests
Integration and e2e tests in Go
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10775229/github-link.png)
Repository with example app and tests
What are we talking about today?
- When unit tests are not enough?
- Good patterns for different scenarios:
- Testing with database
- Testing APIs
- Testing entire app/service
- How to keep tests easy to read, but not dreadful to write?
![](https://media2.giphy.com/media/huh7lUqEG4irK/giphy.gif)
When unit tests are not enough?
When we want to:
- Test how components interact with each other.
- Make sure that setup is correct.
- Ensure that crucial paths of our app/service works under happy conditions.
![](https://media1.giphy.com/media/FvTeCqovnI96U/giphy.gif)
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10809951/pasted-from-clipboard.png)
Testing with database
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10803417/pasted-from-clipboard.png)
version: "3.9"
services:
db:
image: "postgres:15.2-alpine"
environment:
POSTGRES_DB: expense_tracker
POSTGRES_USER: postgres
POSTGRES_PASSWORD: secret123
healthcheck:
test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ]
interval: 3s
timeout: 60s
retries: 10
start_period: 5s
ports:
- "5432"
migrate:
image: "expense_tracker/migrate:latest"
build:
context: ../app_to_test/db
environment:
DB_URL: "postgres://postgres:secret123@db:5432/expense_tracker?sslmode=disable"
depends_on:
db:
condition: service_healthy
func StartDB(t *testing.T, ctx context.Context) DBContainer {
// ...
compose, err := tc.NewDockerCompose(path.Join(path.Dir(filename), "docker-compose-db-only.yaml"))
// ...
err = compose.WaitForService("migrate", wait.ForExit()).Up(ctx)
// ...
dbPort, err := dbContainer.MappedPort(ctx, "5432")
// ...
db, err := sql.Open("pgx", dsn)
return DBContainer{ DB: db, DSN: dsn }
}
func TestExpenseRepo_Add(t *testing.T) {
db := StartDB(t, ctx).DB
expenseRepo := api.NewExpenseRepo(db)
t.Run("successfully adds expense", func(t *testing.T) {
expense := api.Expense{
ID: "c811c5d4-c38a-4f61-932d-d656c203b5f6",
Amount: 123_50,
Category: "food",
Date: time.Date(2020, 9, 5, 0, 0, 0, 0, time.UTC),
Notes: "some notes",
}
err := expenseRepo.Add(expense)
require.NoError(t, err, "could not add expense")
result := getAllExpenses(t, db).FindByID(expense.ID)
if assert.NotNil(t, result, "expense not added") {
assert.Equal(t, expense, *result, "added expense is different")
}
})
}
Testing APIs
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10801968/pasted-from-clipboard.png)
func TestAPI(t *testing.T) {
server := startServer(t, ctx)
t.Run("summarize expenses", func(t *testing.T) {
response, responseBody := call(t, server, http.MethodGet, "/expenses/summarize", "")
assert.Equal(t, http.StatusOK, response.StatusCode, "status code")
expected := getExpectedResponse(t)
// Or use https://github.com/kinbiko/jsonassert
assert.JSONEq(t, expected, responseBody, "response body")
})
t.Run("add expense fails", func(t *testing.T) {
response, responseBody := call(t, server, http.MethodPost, "/expenses/add", getRequest(t))
assert.Equal(t, http.StatusBadRequest, response.StatusCode, "status code")
expected := getExpectedResponse(t)
assert.JSONEq(t, expected, responseBody, "response body")
result := getAllExpenses(t, db).FindByID("...")
assert.Nil(t, result, "expense added")
})
}
func startServer(t *testing.T, ctx context.Context) *httptest.Server {
dbContainer := test_repos.StartDB(t, ctx)
err := os.Setenv("DB_URL", dbContainer.DSN)
// ...
setup, err := api.NewSetup()
// ...
return httptest.NewServer(setup.APIMux)
}
func startServer(t *testing.T, ctx context.Context) *httptest.Server {
// ...
}
func call(
t *testing.T,
srv *httptest.Server,
method, path, body string,
) (*http.Response, string) {
// ...
}
func getRequest(t *testing.T) string {
// ...
}
func getExpectedResponse(t *testing.T) string {
// ...
}
Go testing goodies
How do we get here?
func getRequest(t *testing.T, requestPath string) (string, error) {
t.Helper()
file, err := os.ReadFile(requestPath)
if err != nil {
return "", err
}
return string(file), nil
}
t.Name()
func getRequest(t *testing.T) (string, error) {
t.Helper()
path := fmt.Sprintf("./testdata/%s/request.json", t.Name())
file, err := os.ReadFile(path)
if err != nil {
return "", err
}
return string(file), nil
}
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10795450/pasted-from-clipboard.png)
func getRequest(t *testing.T) (string, error) {
t.Helper()
path := fmt.Sprintf("./testdata/%s/request.json", t.Name())
file, err := os.ReadFile(path)
if err != nil {
return "", err
}
return string(file), nil
}
t.FailNow()
func getRequest(t *testing.T) string {
t.Helper()
path := fmt.Sprintf("./testdata/%s/request.json", t.Name())
file, err := os.ReadFile(path)
require.NoError(t, err, "read file")
return string(file)
}
func startServer(t *testing.T, ctx context.Context) (*httptest.Server, func()) {
t.Helper()
// ...
server := httptest.NewServer(setup.APIMux)
return server, server.Close
}
t.Cleanup()
func startServer(t *testing.T, ctx context.Context) *httptest.Server {
t.Helper()
// ...
server := httptest.NewServer(setup.APIMux)
t.Cleanup(func() {
server.Close()
})
return server
}
t.Cleanup(func() {
// When test fail, printing logs is usually helpful :)
if t.Failed() {
reader, _ := getServerContainer(t, ctx, compose).Logs(ctx)
bytes, _ := io.ReadAll(reader)
fmt.Println(`\nLogs from "server" container:\n`, string(bytes))
}
assert.NoError(t, compose.Down(ctx, tc.RemoveOrphans(true), tc.RemoveImagesLocal))
})
t.Failed()
func TestAPI(t *testing.T) {
server := startServer(t, ctx)
t.Run("add expense fails", func(t *testing.T) {
response, responseBody := call(
t, server, http.MethodPost, "/expenses/add", getRequest(t),
)
assert.Equal(t, http.StatusBadRequest, response.StatusCode, "status code")
expected := getExpectedResponse(t)
assert.JSONEq(t, expected, responseBody, "response body")
})
}
Crispy clear tests, where helpers just disappear
Testing entire app/service
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10796008/pasted-from-clipboard.png)
The original idea came from service like this. It was super annoying to test.
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10796030/pasted-from-clipboard.png)
Let's see how our example app works
And this is a diagram of how test works
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10796061/pasted-from-clipboard.png)
version: "3.9"
services:
server:
image: "expense_tracker/server:latest"
build:
context: ./..
dockerfile: app_to_test/server/Dockerfile
environment:
DB_URL: "postgres://postgres:secret123@db:5432/expense_tracker?sslmode=disable"
BANK_API_URL: "${BANK_API_URL}"
depends_on:
db:
condition: service_healthy
# Thanks to this container can call our mock HTTP server on host machine.
extra_hosts:
- "host.docker.internal:host-gateway"
ports:
# Once again, we use a random port to avoid conflicts.
- "8000"
db:
image: "postgres:15.2-alpine"
environment:
POSTGRES_DB: expense_tracker
POSTGRES_USER: postgres
POSTGRES_PASSWORD: secret123
healthcheck:
test: [ "CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}" ]
interval: 3s
timeout: 60s
retries: 10
start_period: 5s
migrate:
image: "expense_tracker/migrate:latest"
build:
context: ./../app_to_test/db
environment:
DB_URL: "postgres://postgres:secret123@db:5432/expense_tracker?sslmode=disable"
depends_on:
db:
condition: service_healthy
const expenseToSyncID = "677df0c4-d829-42eb-a0c9-29d5b0a2bbe4"
func TestE2E(t *testing.T) {
ctx := context.Background()
bankAPIAddress := mockBankAPI(t)
address := startApp(t, ctx, bankAPIAddress)
t.Run("app is starting properly", func(t *testing.T) {
response, err := http.Get(fmt.Sprintf("%s/expenses/all", address))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, response.StatusCode, "status code")
})
t.Run("sync expenses", func(t *testing.T) {
response, err := http.Get(fmt.Sprintf("%s/expenses/sync", address))
require.NoError(t, err)
require.Equal(t, http.StatusOK, response.StatusCode, "status code")
response, err = http.Get(fmt.Sprintf("%s/expenses/all", address))
require.NoError(t, err)
responseBody, err := io.ReadAll(response.Body)
require.NoError(t, err)
assert.Contains(t, string(responseBody), expenseToSyncID)
})
}
func mockBankAPI(t *testing.T) (address string) {
t.Helper()
mux := http.NewServeMux()
mux.Handle("/get-transactions", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(fmt.Sprintf(`
[
{
"id": "%s",
"amount": 500.00,
"category": "food",
"created_at": "2020-01-01T00:00:00Z"
}
]`,
expenseToSyncID)))
}))
server := httptest.NewUnstartedServer(mux)
listener, err := net.Listen("tcp4", "0.0.0.0:0")
require.NoError(t, err, "could not start listener")
addr, err := net.ResolveTCPAddr(listener.Addr().Network(), listener.Addr().String())
require.NoError(t, err, "could not resolve tcp addr")
server.Listener = listener
server.Start()
t.Cleanup(func() {
server.Close()
})
return fmt.Sprintf("http://host.docker.internal:%d", addr.Port)
}
func startApp(t *testing.T, ctx context.Context, bankAPIAddress string) (address string) {
t.Helper()
compose, err := tc.NewDockerComposeWith(
tc.WithStackFiles("./docker-compose-for-e2e.yaml"),
// Giving unique name to each docker compose stack allows us to run tests in parallel.
tc.StackIdentifier(uuid.New().String()),
)
require.NoError(t, err, "docker compose setup")
t.Cleanup(func() {
// When test fail, printing logs is usually helpful :)
if t.Failed() {
reader, _ := getServerContainer(t, ctx, compose).Logs(ctx)
bytes, _ := io.ReadAll(reader)
fmt.Println(`\nLogs from "server" container:\n`, string(bytes))
}
assert.NoError(t, compose.Down(ctx, tc.RemoveOrphans(true), tc.RemoveImagesLocal))
})
err = compose.
WithEnv(map[string]string{
"BANK_API_URL": bankAPIAddress,
}).
WaitForService("server", wait.ForLog("running...")).
Up(ctx)
require.NoError(t, err, "docker compose up")
// Port is randomly assigned by docker. We need to get it.
apiPort, err := getServerContainer(t, ctx, compose).MappedPort(ctx, "8000")
require.NoError(t, err, "docker compose server port")
return fmt.Sprintf("http://localhost:%s", apiPort.Port())
}
![](https://s3.amazonaws.com/media-p.slid.es/uploads/2524231/images/10775229/github-link.png)
Thanks!
Question for you: Should you start containers within TestMain
?
Any questions for me?
Beyond unit tests
By antosdaniel
Beyond unit tests
- 72