Lecture 7:

IPC. Pipe, FIFO. XSI and POSIX. Sockets: domain, network.

slides.com/gerold103/sysprog_eng7

github.com/sysprogio/sysprog/tree/master/lecture_examples/7_ipc

Version: 3

System programming

Education

Lecture plan

  • IPC definition
  • Pipe and shared mmap
  • FIFO - named pipe
  • XSI message queue
  • XSI semaphore
  • XSI shared mem
  • POSIX semaphore
  • UNIX sockets

Threads

int
pthread_create(pthread_t *thread, const pthread_attr_t *attr,
               void *(*start_routine) (void *), void *arg);

int
pthread_mutex_lock(pthread_mutex_t *mutex);
int
pthread_mutex_unlock(pthread_mutex_t *mutex);

int
pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex);
int
pthread_cond_signal(pthread_cond_t *cond);
int
pthread_cond_broadcast(pthread_cond_t *cond);

Task

.text

.data

.bss

.stack

.heap

IPC: pipe, mmap

Pipe

fdtable

fd[1]

fdtable

fd[0]

Process 1

Process 2

int
pipe(int fildes[2]);

Mmap

Process 1

Process 2

write

read

read
write

read
write

VMA

void *
mmap(void *addr, size_t len,
     int prot, int flags,
     int fd, off_t offset);

IPC: pipe. Simple sort

7_ipc/1_simple_sort.c [1]

struct worker {
	int *array;
	int size;
};

int
cmp(const void *a, const void *b)
{
	return *(int *)a - *(int *)b;
}

void
sorter(struct worker *worker, const char *filename)
{
	FILE *file = fopen(filename, "r");
	int size = 0;
	int capacity = 1024;
	int *array = malloc(capacity * sizeof(int));
	while (fscanf(file, "%d", &array[size]) > 0) {
		++size;
		if (size == capacity) {
			capacity *= 2;
			array = realloc(array, capacity * sizeof(int));
		}
	}
	qsort(array, size, sizeof(int), cmp);
	fclose(file);
	worker->array = array;
	worker->size = size;
}

One file sorter context

Read from one file and sort its data

7_ipc/1_simple_sort.c [2]

int
main(int argc, const char **argv)
{
	struct timespec ts;
	clock_gettime(CLOCK_MONOTONIC, &ts);
	uint64_t start_ns = ts.tv_sec * 1000000000 + ts.tv_nsec;
	int nfiles = argc - 1;
	struct worker *workers = malloc(sizeof(struct worker) * nfiles);
	struct worker *w = workers;
	int total_size = 0;
	for (int i = 0; i < nfiles; ++i, ++w) {
		sorter(w, argv[i + 1]);
		total_size += w->size;
	}
	int *total_array = malloc(total_size * sizeof(int));
	int *pos = total_array;
	w = workers;
	for (int i = 0; i < nfiles; ++i, ++w) {
		memcpy(pos, w->array, w->size * sizeof(int));
		pos += w->size;
	}
	clock_gettime(CLOCK_MONOTONIC, &ts);
	uint64_t end_ns = ts.tv_sec * 1000000000 + ts.tv_nsec;
	double sec = (end_ns - start_ns) / 1000000000.0;
	printf("presort time = %lfs\n", sec);
	return 0;
}

Each file is sorted in its worker

Measure and print the execution time

Merge into the final array

IPC: pipe. Simple sort

7_ipc/1_simple_sort.c

Sort by one process

4 files with 20 000 000 numbers each

~15 sec

IPC: pipe. Parallel sort

7_ipc/2_parallel_sort.c [1]

struct worker {
	int fd[2];
	int *array;
	int size;
	int id;
};

void
sorter(struct worker *worker, const char *filename)
{
	close(worker->fd[0]);
	FILE *file = fopen(filename, "r");
	int size = 0;
	int capacity = 1024;
	int *array = malloc(capacity * sizeof(int));
	while (fscanf(file, "%d", &array[size]) > 0) {
		++size;
		if (size == capacity) {
			capacity *= 2;
			array = realloc(array, capacity * sizeof(int));
		}
	}
	qsort(array, size, sizeof(int), cmp);
	fclose(file);
	printf("Worker %d sorted %d numbers\n", worker->id, size);
	write(worker->fd[1], &size, sizeof(size));
	write(worker->fd[1], array, sizeof(int) * size);
	close(worker->fd[1]);
	free(array);
}

Pipe to transfer data to the parent process, and id for printing

Close unused end of the pipe

Parent listens its fd[0]. Send the result to it through fd[1]. Firstly the number count, and then the numbers

Function to execute in worker processes

IPC: pipe. Parallel sort

Why is number count sent before the numbers?

To be able in the receiver to pre-malloc() a buffer for all the numbers at once without reading them in chunks and doing reallocs().

1 point

printf("Worker %d sorted %d numbers\n", worker->id, size);
write(worker->fd[1], &size, sizeof(size));
write(worker->fd[1], array, sizeof(int) * size);

7_ipc/2_parallel_sort.c [2]

7_ipc/2_parallel_sort.c [3]

int
main(int argc, const char **argv)
{
	struct timespec ts;
	clock_gettime(CLOCK_MONOTONIC, &ts);
	uint64_t start_ns = ts.tv_sec * 1000000000 + ts.tv_nsec;
	int nfiles = argc - 1;
	struct worker *workers = malloc(sizeof(struct worker) * nfiles);
	struct worker *w = workers;
	for (int i = 0; i < nfiles; ++i, ++w) {
		pipe(w->fd);
		w->id = i;
		if (fork() == 0) {
			sorter(w, argv[i + 1]);
			free(workers);
			return 0;
		}
		close(w->fd[1]);
	}
	int total_size = 0;
	w = workers;
	for (int i = 0; i < nfiles; ++i, ++w) {
		read_from_pipe(w->fd[0], (char *) &w->size, sizeof(w->size));
		w->array = malloc(w->size * sizeof(int));
		read_from_pipe(w->fd[0], (char *) w->array,
			       w->size * sizeof(int));
		printf("Got %d numbers from worker %d\n", w->size, w->id);
		close(w->fd[0]);
		wait(NULL);
		total_size += w->size;
	}

Process creates workers and will read their results from

w->fd[0]

Unused w->fd[1] is closed. Unidirectional channel is left from the child to the parent.

Read result from the child process and wait for its termination

7_ipc/2_parallel_sort.c [4]

	int *total_array = malloc(total_size * sizeof(int));
	int *pos = total_array;
	w = workers;
	for (int i = 0; i < nfiles; ++i, ++w) {
		memcpy(pos, w->array, w->size * sizeof(int));
		pos += w->size;
	}
	clock_gettime(CLOCK_MONOTONIC, &ts);
	uint64_t end_ns = ts.tv_sec * 1000000000 + ts.tv_nsec;
	double sec = (end_ns - start_ns) / 1000000000.0;
	printf("presort time = %lfs\n", sec);
	return 0;
}

void
read_from_pipe(int fd, char *in, int size)
{
	int total = 0;
	int rc = read(fd, in, size);
	while (rc > 0 && size > 0) {
		size -= rc;
		in += rc;
		total += rc;
		rc = read(fd, in, size);
	}
}

Merge the arrays into the final result

Calculate time

Big data is consumed via multiple read() calls

IPC: pipe. Parallel sort

7_ipc/2_parallel_sort.c

Sort by multiple processes

4 files by 20 000 000 numbers each

~4.3 sec

4 processes

х 3.4

IPC: mmap. Parallel sort

7_ipc/3_mem_sort.c [1]

#define MEM_SIZE 65536
#define IS_READABLE 0
#define IS_WRITABLE 1
#define MEM_META_SIZE 2

struct worker {
	char *mem;
	int *array;
	int size;
	int id;
};

Parent shares 65536 bytes with each child

First two bytes are reserved to sync parent and child actions

7_ipc/3_mem_sort.c [2]

void
write_to_shared_mem(char *mem, const char *src, int size)
{
	volatile char *is_readable = &mem[IS_READABLE];
	volatile char *is_writable = &mem[IS_WRITABLE];
	mem += MEM_META_SIZE;
	int mem_size = MEM_SIZE - MEM_META_SIZE;
	int saved_size = mem_size;
	char *saved_mem = mem;
	while (1) {
		while (! __atomic_load_n(is_writable, __ATOMIC_ACQUIRE))
			sched_yield();
		int to_copy = mem_size > size ? size : mem_size;
		memcpy(mem, src, to_copy);
		size -= to_copy;
		mem_size -= to_copy;
		mem += to_copy;
		src += to_copy;

		__atomic_store_n(is_writable, 0, __ATOMIC_RELEASE);
		__atomic_store_n(is_readable, 1, __ATOMIC_RELEASE);
		if (size == 0)
			break;
		mem = saved_mem;
		mem_size = saved_size;
	}
}

First two bytes are reserved to sync read/write

Wait until the reader is gone

Write as much as possible

If there is more to write, then wait until the current data is consumed by the reader

7_ipc/3_mem_sort.c [3]

void
read_from_shared_mem(char *mem, char *dst, int size)
{
	volatile char *is_readable = &mem[IS_READABLE];
	volatile char *is_writable = &mem[IS_WRITABLE];
	mem += MEM_META_SIZE;
	int mem_size = MEM_SIZE - MEM_META_SIZE;
	int saved_size = mem_size;
	char *saved_mem = mem;
	while (1) {
		while (! __atomic_load_n(is_readable, __ATOMIC_ACQUIRE))
			sched_yield();
		int to_copy = mem_size > size ? size : mem_size;
		memcpy(dst, mem, to_copy);
		size -= to_copy;
		mem_size -= to_copy;
		mem += to_copy;
		dst += to_copy;

		__atomic_store_n(is_readable, 0, __ATOMIC_RELEASE);
		__atomic_store_n(is_writable, 1, __ATOMIC_RELEASE);
		if (size == 0)
			break;
		mem = saved_mem;
		mem_size = saved_size;
	}
}

Wait until the writer is gone

Read as much as possible

If there is more to read, then wait until new data appears

7_ipc/3_mem_sort.c [4]

void
sorter(struct worker *worker, const char *filename)
{
	FILE *file = fopen(filename, "r");
	int size = 0;
	int capacity = 1024;
	int *array = malloc(capacity * sizeof(int));
	while (fscanf(file, "%d", &array[size]) > 0) {
		++size;
		if (size == capacity) {
			capacity *= 2;
			array = realloc(array, capacity * sizeof(int));
		}
	}
	qsort(array, size, sizeof(int), cmp);
	fclose(file);
	printf("Worker %d sorted %d numbers\n", worker->id, size);
	write_to_shared_mem(worker->mem, (char *) &size, sizeof(size));
	write_to_shared_mem(worker->mem, (char *) array, sizeof(int) * size);
	free(array);
}

Firstly send number count to the parent, so as it could do malloc() of the appropriate size. Then send the numbers

7_ipc/3_mem_sort.c [5]

int
main(int argc, const char **argv)
{
	struct timespec ts;
	clock_gettime(CLOCK_MONOTONIC, &ts);
	uint64_t start_ns = ts.tv_sec * 1000000000 + ts.tv_nsec;
	int nfiles = argc - 1;
	struct worker *workers = malloc(sizeof(struct worker) * nfiles);
	struct worker *w = workers;
	for (int i = 0; i < nfiles; ++i, ++w) {
		w->id = i;
		w->mem = mmap(NULL, MEM_SIZE, PROT_READ | PROT_WRITE,
			      MAP_ANON | MAP_SHARED, -1, 0);
		w->mem[IS_READABLE] = 0;
		w->mem[IS_WRITABLE] = 1;
		if (fork() == 0) {
			sorter(w, argv[i + 1]);
			free(workers);
			return 0;
		}
	}
	int total_size = 0;
	w = workers;
	for (int i = 0; i < nfiles; ++i, ++w) {
		read_from_shared_mem(w->mem, (char *) &w->size,
				     sizeof(w->size));
		w->array = malloc(w->size * sizeof(int));
		read_from_shared_mem(w->mem, (char *) w->array,
				     w->size * sizeof(int));
		printf("Got %d numbers from worker %d\n", w->size, w->id);
		wait(NULL);
		total_size += w->size;
	}

Each worker has a memory segment shared with the parent

Read the number count, and the numbers

7_ipc/3_mem_sort.c [6]

	int *total_array = malloc(total_size * sizeof(int));
	int *pos = total_array;
	w = workers;
	for (int i = 0; i < nfiles; ++i, ++w) {
		memcpy(pos, w->array, w->size * sizeof(int));
		pos += w->size;
	}
	clock_gettime(CLOCK_MONOTONIC, &ts);
	uint64_t end_ns = ts.tv_sec * 1000000000 + ts.tv_nsec;
	double sec = (end_ns - start_ns) / 1000000000.0;
	printf("presort time = %lfs\n", sec);
	return 0;
}

Merge the result array

Calculate time

IPC: mmap. Parallel sort

void
write_to_shared_mem(char *mem, ...)
{
	volatile char *is_readable = &mem[IS_READABLE];
	volatile char *is_writable = &mem[IS_WRITABLE];
/* ... */

void
read_from_shared_mem(char *mem, ...)
{
	volatile char *is_readable = &mem[IS_READABLE];
	volatile char *is_writable = &mem[IS_WRITABLE];
/* ... */

Why mmap sort implementation uses volatile flags and atomics?

Volatile - so as the compiler would not optimize these variables. They can be changed from another process. Atomics are used to prevent operation reordering on the CPU

1 point

IPC: pipe, mmap

init/lauchd

Process family tree

Can't create and share anonymous resources. How to deal with it?

Can share anon mmap, pipe

IPC: FIFO

int
mkfifo(const char *path, mode_t mode);

Process 1

Process 2

read
write

read
write

FIFO

Process N

read
write

Process 3

read
write

...

IPC: FIFO. Server

int main()
{
	mkfifo("/tmp/fifo_server", S_IRWXU | S_IRWXO);
	int fd = open("/tmp/fifo_server", O_RDONLY);
	while (1) {
		pid_t new_client;
		if (read(fd, &new_client, sizeof(pid_t)) > 0)
			printf("new client %d\n", (int) new_client);
		else
			sched_yield();
	}
}

7_ipc/4_fifo_server.c

Server creates a FIFO channel with read/write rights for everyone

After the channel creation it can be used like a file. But it does not store anything on a disk. The data is in the kernel, in the main memory

Clients write their pid. The server reads them and prints

IPC: FIFO. Client

int main()
{
	int fd = open("/tmp/fifo_server", O_WRONLY);
	pid_t pid = getpid();
	write(fd, &pid, sizeof(pid));
	printf("my pid %d is sent to server\n", (int) pid);
	close(fd);
	return 0;
}

7_ipc/4_fifo_client.c

Client opens the FIFO like a file

Writes pid

IPC: FIFO

$> gcc 4_fifo_server.c -o server

$> ./server
$> gcc 4_fifo_client.c -o client
$> ./client
my pid 62686 is sent to server
new client 62686
$> ./client
my pid 62690 is sent to server
new client 62690
$> ./client
my pid 62692 is sent to server
new client 62692
$> ls -al /tmp/fifo_server
prwx---r-x  1 v.shpilevoy /tmp/fifo_server
$> echo '123' > /tmp/fifo_server

               ...
$> cat /tmp/fifo_server
123
$>
$>

XSI IPC

X/Open System Interfaces

Standard for named IPC objects

  • Message queue
  • Semaphores
  • Shared memory
int
msgget(key_t key, int msgflg);

int
semget(key_t key, int nsems, int semflg);

int
shmget(key_t key, size_t size, int shmflg);

Naming by special values of type key_t

XSI IPC. Naming [1]

key_t
ftok(const char *path, int id);
key_t
ftok(const char *pathname, int proj_id)
{
        struct stat st;
        if (stat(pathname, &st) < 0)
                return (key_t) -1;

        key_t key = ((st.st_ino & 0xffff) | ((st.st_dev & 0xff) << 16)
                    | ((proj_id & 0xff) << 24));
        return key;
}

Key is generated by a path in a file system, and an arbitrary number

XSI IPC. Naming [2]

void
try_key(const char *path, int id)
{
	key_t key = ftok(path, id);
	printf("key(\"%s\", %d) = ", path, id);
	if (key == -1)
		printf("%s\n", strerror(errno));
	else
		printf("%d\n", (int) key);
}

int main(int argc, const char **argv)
{
	try_key("/not/exising/path", 100);
	try_key(argv[0], 100);
	try_key(argv[0], 101);
	try_key(argv[0], 101 + 1024);
	return 0;
}
$> gcc 5_key_t.c

$> ./a.out

$> key("/not/exising/path", 100) =
        No such file or directory
key("./a.out", 100) = 1678028750
key("./a.out", 101)  = 1694805966
key("./a.out", 1125) = 1694805966

7_ipc/5_key_t.c

Only 8 bits are used from the second argument

Path in ftok is a temporary file; or the executable file (argv[0]); or any other file. But the file should exist. The second argument is to reduce collision likelyhood

XSI IPC. Message queue [1]

Process 1

Process 2

write

read

Type t1

Type t2

Type t3

Type tN

int
msgget(key_t key, int msgflg);

int
msgctl(int msqid, int cmd, struct msqid_ds *buf);

int
msgsnd(int msqid, const void *msgp, size_t msgsz,
       int msgflg);

ssize_t
msgrcv(int msqid, void *msgp, size_t msgsz,
       long msgtyp, int msgflg);

Creation of a queue, or connection to an existing one

Queue deletion; its metadata retrieval

Send to/read from a queue

Message type can be used to read only certain messages

Type is stored in message body in first several bytes

$> cat /proc/sys/kernel/msgmax 
8192

$> cat /proc/sys/kernel/msgmnb
16384

Maximal message and queue size, in bytes

XSI IPC. Message queue [2]

int main(int argc, const char **argv)
{
	key_t key = ftok(argv[0], 0);
	int queue_id = msgget(key, IPC_CREAT | S_IRWXU | S_IRWXO);
	printf("key = %d, queue_id = %d\n", (int) key, queue_id);

	struct msqid_ds queue_stat;
	msgctl(queue_id, IPC_STAT, &queue_stat);
	printf("message count = %d\n", (int) queue_stat.msg_qnum);
	printf("max queue bytes = %d\n", (int) queue_stat.msg_qbytes);

	int rc = msgget(key, IPC_CREAT | IPC_EXCL);
	if (rc == -1)
		printf("second msgget returned '%s'\n", strerror(errno));

	msgctl(queue_id, IPC_RMID, NULL);

#ifdef IPC_INFO
	printf("\nSystem wide settings:\n");
	struct msginfo info;
	msgctl(0, IPC_INFO, (struct msqid_ds *) &info);

	printf("max message size = %d\n", info.msgmax);
	printf("max queue bytes = %d\n", info.msgmnb);
	printf("max number of message queues = %d\n", info.msgmni);
#endif
	return 0;
}

7_ipc/6_msgget.c

Create a queue with enabled read/write

Look at its characteristics via msgctl IPC_STAT

Ensure that IPC_EXCL | IPC_CREAT returns an error if the queue exists already

Delete the queue by msgctl IPC_RMID

Linux has command IPC_INFO to look at global limits of all queues

XSI IPC. Message queue [3]

7_ipc/6_msgget.c

$> gcc 6_msgget.c

$> ./a.out
key = 311359, queue_id = 1245184
message count = 0
max queue bytes = 2048
second msgget returned 'File exists'

Mac

Linux

$> gcc 6_msgget.c

$> ./a.out
key = 3014816, queue_id = 327680
message count = 0
max queue bytes = 16384
second msgget returned 'File exists'

System wide settings:
max message size = 8192
max queue bytes = 16384
max number of message queues = 32000

XSI IPC. Message queue [4]. Client

7_ipc/7_msgsend_client.c [1]

int
pack_msg(char *msg, long type, const char *data, int data_size)
{
	char *pos = msg;
	memcpy(pos, &type, sizeof(type));
	pos += sizeof(type);
	memcpy(pos, &data_size, sizeof(data_size));
	pos += sizeof(data_size);
	memcpy(pos, data, data_size);
	pos += data_size;
	return pos - msg;
}

void
send_msg(int queue_id, long type, const char *data,
         int data_size)
{
	char msg[512];
	int size = pack_msg(msg, type, data, data_size);
	msgsnd(queue_id, msg, size, 0);
	printf("sent %d bytes\n", size);
}

Encode a message with a given type

Type is encoded into first sizeof(long) bytes

Data size is encoded into next sizeof(int) bytes

Data is in the tail

type

size

data

sizeof(long)

sizeof(int)

size

Sender function encodes a message into a local buffer and puts into the queue

7_ipc/7_msgsend_client.c [2]

int
main(int argc, const char **argv)
{
	key_t key = ftok("./server", 0);
	int queue_id = msgget(key, 0);
	printf("connected to queue %d\n", queue_id);

	const char *text = "hello, world";
	send_msg(queue_id, 100, text, strlen(text) + 1);

	text = "hello, world of type 50";
	send_msg(queue_id, 50, text, strlen(text) + 1);

	text = "hello, world of type 60";
	send_msg(queue_id, 60, text, strlen(text) + 1);
	return 0;
}

Attach to a queue, created by the server

Send messages with types 100, 50, 60. Exactly in that order

XSI IPC. Message queue [5]. Server

7_ipc/7_msgsend_server.c [1]

int
unpack_msg(const char *msg, int size, long *type, const char **data,
	   int *data_size)
{
	const char *pos = msg;
	memcpy(type, pos, sizeof(*type));
	pos += sizeof(type);
	memcpy(data_size, pos, sizeof(*data_size));
	pos += sizeof(*data_size);
	*data = pos;
	pos += *data_size;
	return pos - msg;
}

void
recv_msg(int queue_id, long type)
{
	char msg[512];
	ssize_t recv = msgrcv(queue_id, msg, sizeof(msg), type, 0);
	printf("received %d\n", (int) recv);
	const char *data;
	int data_size;
	unpack_msg(msg, recv, &type, &data, &data_size);
	printf("type = %ld, size = %d, str = %s\n", type, data_size, data);
}

Decode a message. Just like it was encoded

Extract first sizeof(long) bytes as a type

Extract data size from next sizeof(int) bytes

The rest is data

Receive a message from the queue and print it

7_ipc/7_msgsend_server.c [2]

int main(int argc, const char **argv)
{
	key_t key = ftok("./server", 0);
	int queue_id = msgget(key, IPC_CREAT | S_IRWXU | S_IRWXO);
	printf("connected to queue %d\n", queue_id);

	while (1) {
		struct msqid_ds queue_stat;
		if (msgctl(queue_id, IPC_STAT, &queue_stat) != 0) {
			printf("error = %s\n", strerror(errno));
			msgctl(queue_id, IPC_RMID, NULL);
			return -1;
		}
		if (queue_stat.msg_qnum > 0) {
			printf("message count = %d\n",
			       (int) queue_stat.msg_qnum);
			break;
		}
		sched_yield();
	}

	recv_msg(queue_id, 0);

	recv_msg(queue_id, 60);

	recv_msg(queue_id, -200);

	msgctl(queue_id, IPC_RMID, NULL);
	return 0;
}

Create a queue. Now clients can connect and put messages in it

In a loop wait until new messages appear

Using IPC_STAT get the queue meta and check message count

A client is found. Read 3 messages

Type 0 means 'read any message, respecting the queue order'

Type > 0 is read of a message of only that type. Here a message of type 60 is received out of queue order

Type < 0 - is read of a message of any type < |type|

XSI IPC. Message queue [6]. Server

7_ipc/7_msgsend_server.c

$> gcc 7_msgsend_server.c -o server

$> ./server
connected to queue 1310720
$> gcc 7_msgsend_client.c -o client
message count = 1
received 25
type = 100, size = 13, str = hello, world
received 36
type = 60, size = 24, str = hello, world of type 60
received 36
type = 50, size = 24, str = hello, world of type 50
$> ./client
connected to queue 1310720
sent 25 bytes
sent 36 bytes
sent 36 bytes

The message is received out of queue order, because its type was specified explicitly

XSI IPC. Message queue [7]. Server

7_ipc/7_msgsend_server_flag.c [1]

void
recv_msg(int queue_id, long type)
{
	char msg[512];
	ssize_t recv;
	while(1) {
		recv = msgrcv(queue_id, msg, sizeof(msg), type, IPC_NOWAIT);
		if (recv != -1)
			break;
		if (errno != ENOMSG) {
			printf("error = %s\n", strerror(errno));
			return;
		}
	}
	printf("received %d\n", (int) recv);
	const char *data;
	int data_size;
	unpack_msg(msg, recv, &type, &data, &data_size);
	printf("type = %ld, size = %d, str = %s\n", type, data_size, data);
}

When IPC_NOWAIT is used, msgrcv() does not block in case there are no messages

Read again and again until a message is received successfully

7_ipc/7_msgsend_server_flag.c [2]

int main(int argc, const char **argv)
{
	key_t key = ftok("./server", 0);
	int queue_id = msgget(key, IPC_CREAT | S_IRWXU | S_IRWXO);
	printf("connected to queue %d\n", queue_id);

	recv_msg(queue_id, 0);

	recv_msg(queue_id, 60);

	recv_msg(queue_id, -200);

	msgctl(queue_id, IPC_RMID, NULL);
	return 0;
}

No more waits - recv_msg does everything

XSI IPC. ipcs, ipcrm

$> ipcs
------ Message Queues --------
key        msqid      owner      perms      used-bytes   messages    

------ Shared Memory Segments --------
key        shmid      owner      perms      bytes      nattch     status      
0x00000000 196608     vladislav  600        33554432   2          dest         
0x00000000 163841     vladislav  600        16777216   2          dest         
0x00000000 294914     vladislav  600        524288     2          dest         
0x00000000 393219     vladislav  600        524288     2          dest         
0x00000000 425988     vladislav  600        524288     2          dest         
0x00000000 753669     vladislav  600        524288     2          dest         
0x00000000 983046     vladislav  600        524288     2          dest         
0x002e0094 1015815    root       707        1024       0                       
0x002e0095 1048584    root       707        1024       0                       

------ Semaphore Arrays --------
key        semid      owner      perms      nsems

$> ipcrm -m 1048584

Delete shared memory via ipcrm

XSI IPC. Semaphore [1]

struct semaphore {
        int counter;
};

void
lock(struct semaphore *sem)
{
        if (sem->counter == 0)
                wait_not_0();
        sem->counter--;
}

void
unlock(struct semaphore *sem)
{
        sem->counter++;
        wakeup_waiters();
}

XSI IPC. Semaphore [2]

int
semget(key_t key, int nsems, int semflg);

int
semctl(int semid, int semnum, int cmd, ...);

int
semop(int semid, struct sembuf *sops, size_t nsops);

Create a semaphore array of size nsems. Flags just like with queues: IPC_CREAT, IPC_EXCL, permissions

Control operations on a semaphore array. cmd is a command:

  • IPC_RMID - delete the array
  • GETALL/SETALL - get/set semaphore array values to the 4th argument values
  • GETVAL/SETVAL - get/set value of one semaphore by index semnum
  • ...

Atomically apply an array of operations to the array of semaphores. Each operation:

struct sembuf {
        unsigned short sem_num;
        short sem_op;
        short sem_flg;
};

- to which semaphore apply

- how much to add/subtract

- flags. For example, SEM_UNDO

XSI IPC. Semaphore [3]

7_ipc/8_sem.c [1]

void
print_all_sems(int semid)
{
	unsigned short values[3];
	if (semctl(semid, -1, GETALL, values) == -1) {
		printf("error = %s\n", strerror(errno));
	} else {
		printf("Values are %d %d %d\n", values[0], values[1],
		       values[2]);
	}
}

void
init_sems(int semid)
{
	unsigned short values[3] = {1, 2, 3};
	semctl(semid, -1, SETALL, values);
	printf("Sem array is initialized\n");
	print_all_sems(semid);
}

A function to initialize an array of 3 semaphores with values 1, 2, 3 using semctl SETALL

Print all values using semctl GETALL

7_ipc/8_sem.c [2]

int
main()
{
	key_t key = ftok("./a.out", 0);
	int semid = semget(key, 3, IPC_CREAT | IPC_EXCL | S_IRWXU | S_IRWXO);
	if (semid == -1) {
		if (errno != EEXIST) {
			printf("error = %s\n", strerror(errno));
			return -1;
		}
		printf("Sem array already exists\n");
		semid = semget(key, 0, 0);
		if (semid == -1) {
			printf("error = %s\n", strerror(errno));
			return -1;
		}
		printf("Connected to %d\n", semid);
	} else {
		printf("Created sems with id %d\n", semid);
		init_sems(semid);
	}

	struct sembuf ops[100], *op;
	int ops_count = 0, sem_num;
	bool is_inside_txn = false;
	char cmd, *line = NULL;
	size_t line_size = 0;
	while (1) {
		getline(&line, &line_size, stdin);
		if (line == NULL)
			continue;
		/* Trim '\n'. */
		line[strlen(line) - 1] = 0;

New process tries to create semaphores

If didn't work because they already exist, then print it and connect

If succeeded, then initialize

Beginning of a command parser: state, reading of lines from stdin

7_ipc/8_sem.c [3]

                if (strcmp(line, "begin") == 0) {
			is_inside_txn = true;
		} else if (strcmp(line, "commit") == 0) {
			if (semop(semid, ops, ops_count) == -1) {
				printf("error = %s\n", strerror(errno));
			} else {
				is_inside_txn = false;
				ops_count = 0;
				print_all_sems(semid);
			}
		} else if (strcmp(line, "rollback") == 0) {
			is_inside_txn = false;
			ops_count = 0;
		} else if (strcmp(line, "delete") == 0) {
			if (semctl(semid, -1, IPC_RMID) == -1)
				printf("error = %s\n", strerror(errno));
			return 0;
		} else if (strcmp(line, "quit") == 0) {
			return 0;
		} else if (strcmp(line, "show") == 0) {
			print_all_sems(semid);
		} else {

Main commands

"begin/commit" allow to accumulate many commands and apply at once in semop()

"show" prints semaphore array content

7_ipc/8_sem.c [4]

                        sscanf(line, "%d %c", &sem_num, &cmd);
			op = &ops[ops_count];
			if (cmd == '-') {
				op->sem_op = -1;
			} else if (cmd == '+') {
				op->sem_op = 1;
			} else {
				printf("Unknown operation\n");
				continue;
			}
			op->sem_num = sem_num - 1;
			op->sem_flg = SEM_UNDO;
			if (! is_inside_txn) {
				if (semop(semid, op, 1) == -1)
					printf("error = %s\n", strerror(errno));
				else
					print_all_sems(semid);
			} else {
				ops_count++;
			}
		}
	}
        return 0;
}

Semaphore operation parser

"<semnum> +" - increment semaphore[semnum]

"<semnum> -" - decrement semaphore[semnum]

XSI IPC. Semaphore [4]

7_ipc/8_sem.c

$> gcc 8_sem.c

$> ./a.out
Created sems with id 327692
Sem array is initialized
Values are 1 2 3
$> ./a.out
Sem array already exists
Connected to 327692
2 -
Values are 1 1 3
2 -
Values are 1 0 3
show
Values are 1 0 3
^C

$>
show
Values are 1 2 3
1 -
Values are 0 2 3
$> ./a.out
Sem array already exists
Connected to 327692
begin
1 -
2 -
2 -
commit
          ...
1 +
Values are 1 2 3
Values are 0 0 3

New user connected to existing semaphores

Semaphore values can be decreased

It is visible to everyone

The flag SEM_UNDO helps to safely terminate the process

All locks are returned

Lets make one of the semaphores 0

Another process is blocked on that by a transaction with several operations

The process is unblocked when the transaction can be applied whole

XSI IPC. Semaphore + queue [1]

Type 1

Type 2

Type 3

Type 1

Type 2

Type 3

Source process 1

Source process 2

Source process 3

Destination processes

The task - synchronize access to the queue so as a destination process could atomically read 3 messages at once, 1 from each source. There can be multiple destination processes.

semaphores[0 - 2] = message_count_of_the_process;
semaphores[3] = queue_mutex;

XSI IPC. Semaphore + queue [2]

int sem_get_or_create(key_t key)
{
	int semid = semget(key, 4, IPC_CREAT | IPC_EXCL | S_IRWXU | S_IRWXO);
	if (semid == -1) {
		if (errno != EEXIST) {
			printf("error = %s\n", strerror(errno));
			return -1;
		}
		semid = semget(key, 0, 0);
		if (semid == -1) {
			printf("error = %s\n", strerror(errno));
			return -1;
		}
		printf("Connected to sems %d\n", semid);
	} else {
		printf("Created sems with id %d\n", semid);
		unsigned short values[4] = {0, 0, 0, 1};
		semctl(semid, -1, SETALL, values);
	}
	return semid;
}

void sem_inc(int semid, int myid)
{
	struct sembuf op;
	op.sem_flg = SEM_UNDO;
	op.sem_num = myid - 1;
	op.sem_op = 1;
	semop(semid, &op, 1);
}

7_ipc/9_sem_event_source.c [1]

Semaphores creation

Three message counters, all 0. Mutex on the queue is free (semaphore[3] = 1)

A function to increase a semaphore value via semop()

int
msg_queue_get_or_create(key_t key)
{
	int queueid = msgget(key, IPC_CREAT | S_IRWXU | S_IRWXO);
	if (queueid == -1)
		printf("error = %s\n", strerror(errno));
	else
		printf("Created/connected to queue %d\n", queueid);
	return queueid;
}

int
pack_msg(char *msg, long type, const char *data, int data_size)
{
	char *pos = msg;
	memcpy(pos, &type, sizeof(type));
	pos += sizeof(type);
	memcpy(pos, &data_size, sizeof(data_size));
	pos += sizeof(data_size);
	memcpy(pos, data, data_size);
	pos += data_size;
	return pos - msg;
}

void
send_msg(int queueid, int myid, const char *data, int data_size)
{
	char msg[512];
	int size = pack_msg(msg, myid, data, data_size);
	msgsnd(queueid, msg, size, 0);
	printf("Sent %d bytes\n", size);
}

7_ipc/9_sem_event_source.c [2]

Creation or connection to a queue (no IPC_EXCL)

Pack and send a message

int
main(int argc, const char **argv)
{
	if (argc < 2) {
		printf("No id\n");
		return -1;
	}
	key_t key = ftok("./event_gen", 0);
	int semid = sem_get_or_create(key);
	if (semid == -1)
		return -1;
	int queueid = msg_queue_get_or_create(key);
	if (queueid == -1)
		return -1;

	int myid = atoi(argv[1]);
	
	char *line = NULL;
	size_t line_size = 0;
	while(1) {
		if (getline(&line, &line_size, stdin) <= 0)
			break;
		send_msg(queueid, myid, line, strlen(line));
		sem_inc(semid, myid);
	}
	if (myid == 1) {
		msgctl(queueid, IPC_RMID, NULL);
		semctl(semid, -1, IPC_RMID);
	}
	return 0;
}

7_ipc/9_sem_event_source.c [3]

The same key can be used for different IPCs

Each source process has a number to label all its messages, from 1 to 3

Increments its semaphore on each send

XSI IPC. Semaphore + queue [3]

int unpack_msg(const char *msg, int size, long *type, const char **data,
	       int *data_size)
{
	const char *pos = msg;
	memcpy(type, pos, sizeof(*type));
	pos += sizeof(type);
	memcpy(data_size, pos, sizeof(*data_size));
	pos += sizeof(*data_size);
	*data = pos;
	pos += *data_size;
	return pos - msg;
}

void recv_msg(int queueid, long type)
{
	char msg[512];
	ssize_t recv = msgrcv(queueid, msg, sizeof(msg), type, 0);
	printf("Received %d\n", (int) recv);
	const char *data;
	int data_size;
	unpack_msg(msg, recv, &type, &data, &data_size);
	printf("Type = %ld, size = %d, str = %s\n", type, data_size, data);
}

void read_data(int queueid)
{
	for (int i = 0; i < 3; ++i)
		recv_msg(queueid, i + 1);
}

7_ipc/9_sem_event_cons.c [1]

Receipt of a message, unpacking

Message consumer reads 1 message from each source

void unlock_events(int semid)
{
	struct sembuf op;
	op.sem_flg = SEM_UNDO;
	op.sem_num = 3;
	op.sem_op = 1;
	semop(semid, &op, 1);
}

void lock_events(int semid)
{
	struct sembuf op[4];
	for (int i = 0; i < 4; ++i) {
		op[i].sem_flg = SEM_UNDO;
		op[i].sem_num = i;
		op[i].sem_op = -1;
	}
	semop(semid, op, 4);
}

int main()
{
	key_t key = ftok("./event_gen", 0);
	int semid = semget(key, 4, 0);
	int queueid = msgget(key, 0);
	while(1) {
		lock_events(semid);
		read_data(queueid);
		unlock_events(semid);
	}
	return 0;
}

7_ipc/9_sem_event_cons.c [2]

To get unique access to the queue with at least 3 different messages all the semaphores are decremented. That will block until all of them > 0

Unlock the mutex, but don't touch the message counters. They were taken and are not put back

Processing loop: lock, read a triplet, unlock

XSI IPC. Semaphore + queue [4]



$> ./event_gen 2
Connected to sems 393229
Created/connected to queue 1376256
$> gcc 9_sem_event_source.c -o
        event_gen
$> ./event_gen 1
Created sems with id 393229
Created/connected to queue 1376256


$> ./event_gen 3
Connected to sems 393229
Created/connected to queue 1376256
$> gcc 9_sem_event_cons.c

$> ./a.out


$> ./a.out
msg 1 from 1
Sent 25 bytes
msg 1 from 2
Sent 25 bytes
msg 1 from 3
Sent 25 bytes
Received 25
Type = 1, size = 13, str = msg 1 from 1

Received 25
Type = 2, size = 13, str = msg 1 from 2

Received 25
Type = 3, size = 13, str = msg 1 from 3
msg 2 from 3
Sent 25 bytes
msg 3 from 3
Sent 25 bytes
msg 4 from 3
Sent 25 bytes
msg 2 from 2
Sent 25 bytes
msg 3 from 2
Sent 25 bytes
msg 4 from 2
Sent 25 bytes
msg 2 from 1
Sent 25 bytes
Received 25
Type = 1, size = 13, str = msg 2 from 1

Received 25
Type = 2, size = 13, str = msg 2 from 2

Received 25
Type = 3, size = 13, str = msg 2 from 3

XSI IPC. Semaphore via mutex

struct semaphore {
	int counter;
	pthread_mutex_t mutex;
	pthread_cond_t cond;
};

static inline void
semaphore_get(struct semaphore *sem)
{
	pthread_mutex_lock(&sem->mutex);
	while (sem->counter == 0)
		pthread_cond_wait(&sem->cond, &sem->mutex);
	sem->counter--;
	pthread_mutex_unlock(&sem->mutex);
}

static inline void
semaphore_put(struct semaphore *sem)
{
	pthread_mutex_lock(&sem->mutex);
	sem->counter++;
	pthread_cond_signal(&sem->cond);
	pthread_mutex_unlock(&sem->mutex);
}

7_ipc/10_sem_mutex.h

XSI IPC. Shared memory [1]

int
shmget(key_t key, size_t size, int shmflg);

int
shmctl(int shmid, int cmd, struct shmid_ds *buf);

void *
shmat(int shmid, const void *shmaddr, int shmflg);

int
shmdt(const void *shmaddr);

Creation/attachment similar to semaphores and queues

Attributes retrieval, memory deletion

shmget() only creates a descriptor. To get it a process needs to call shmat() - analogue of mmap()

Analogue of munmap()

XSI IPC. Shared memory [2]

int
main()
{
	key_t key = ftok("./a.out", 0);
	int id = shmget(key, 1024, IPC_CREAT | S_IRWXU | S_IRWXO);
	printf("Created mem with id %d\n", id);
	char *mem = shmat(id, NULL, 0);
	int align = __alignof__(pthread_mutex_t);
	pthread_mutex_t *mutex = (pthread_mutex_t *)
		(mem + align - (uint64_t)mem % align);
	volatile char *data = (char *) mutex + sizeof(*mutex);
	*data = 0;
	pthread_mutexattr_t attr;
	pthread_mutexattr_init(&attr);
	pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST);
	pthread_mutex_init(mutex, &attr);
	pthread_mutexattr_destroy(&attr);
	printf("Created mutex between processes\n");

	if (fork() == 0) {
		printf("Child holds a lock\n");
		pthread_mutex_lock(mutex);
		*data = 1;
		printf("And the child dies\n");
		exit(1);
	}
	printf("Parent waits for 1 in shmem\n");
	wait(NULL);
	assert(*data == 1);

7_ipc/11_shm.c [1]

Create a memory descriptor

Get the memory

Use first bytes as memory for mutex

Create a new process, which takes the lock, but dies before unlock

Parent waits until the mutex is broken

	printf("Parent tries to lock\n");
	if (pthread_mutex_lock(mutex) == EOWNERDEAD) {
		printf("Owner is dead, restore\n");
		pthread_mutex_consistent(mutex);
	}
	printf("Destroy mutex\n");
	pthread_mutex_unlock(mutex);
	pthread_mutex_destroy(mutex);

	printf("Free shmem\n");
	shmdt(mem);
	shmctl(id, IPC_RMID, NULL);
	return 0;
}

7_ipc/11_shm.c [2]

Mutex is broken

In the beginning the mutex had attribute "robust". It means, that it can be repaired by function consistent().

POSIX IPC. Semaphore [1]

sem_t *
sem_open(const char *name, int oflag,
         mode_t mode, unsigned int value);

int
sem_init(sem_t *sem, int pshared, unsigned int value);

int
sem_wait(sem_t *sem);

int
sem_post(sem_t *sem);

int
sem_unlink(const char *name);

int
sem_close(sem_t *sem);

Creation of a named semaphore. Like in XSI. Does not require existence of a file

Creation of an anonymous semaphore. Will be available only in this process and in children

Subtract 1 from semaphore if it is > 0 or block until it becomes > 0, and then subtract

Add 1 to semaphore, wakeup waiters

Delete the semaphore. Real deletion will happen only after the last user does unlink().

Close but do not delete. Will free local resources in the caller process.

No destructor similar to SEM_UNDO

POSIX IPC. Semaphore [2]

#define WORKER_COUNT 10
sem_t *sem;

void *worker_f(void *arg)
{
	int id = (int) arg;
	while (1) {
		sem_wait(sem);
		printf("Thread %d got the sem\n", id);
		sleep(3);
		sem_post(sem);
		sleep(1);
	}
}

int main()
{
	pthread_t workers[WORKER_COUNT];
	sem = sem_open("/my_sem", O_CREAT, O_RDWR, 5);
	if (sem == SEM_FAILED) {
		printf("error = %s\n", strerror(errno));
		return -1;
	}
	for (int i = 0; i < WORKER_COUNT; ++i)
		pthread_create(&workers[i], NULL, worker_f, (void *) i);
	getchar();
	sem_close(sem);
	return 0;
}

Threads contend for the semaphore. 10 threads per semaphore with initial value 5

Create a named semaphore by a fake path

Create the threads and finish the process when Enter is pressed

7_ipc/12_sem_posix.c

POSIX IPC. Semaphore [3]

7_ipc/12_sem_posix.c

$> gcc 8_sem_posix.c

$> ./a.out
Thread 0 got the sem
Thread 1 got the sem
Thread 2 got the sem
Thread 3 got the sem
Thread 4 got the sem

Thread 5 got the sem
Thread 6 got the sem
Thread 7 got the sem
Thread 8 got the sem
Thread 9 got the sem

Thread 3 got the sem
Thread 1 got the sem
Thread 4 got the sem
Thread 2 got the sem
Thread 0 got the sem

...

IPC. UNIX domain sockets [1]

int
socketpair(int domain, int type, int protocol,
           int socket_vector[2]);

Works like bidirectional pipe - can write/read at both ends

IPC. UNIX domain sockets [2]

int
socketpair(int domain, int type, int protocol,
           int socket_vector[2]);

Protocol - an agreement on the data format and processing

An example of protocol

"Lets send data in packets, each at most 512 bytes. First 4 bytes - packet size in big-endian, next is message type, ... If packets arrive out of order, then sort on the receiver, ...

$> cat /etc/protocols
ip	0	IP		# internet protocol, pseudo protocol number
hopopt	0	HOPOPT		# IPv6 Hop-by-Hop Option [RFC1883]
icmp	1	ICMP		# internet control message protocol
igmp	2	IGMP		# Internet Group Management
ggp	3	GGP		# gateway-gateway protocol
ipencap	4	IP-ENCAP	# IP encapsulated in IP (officially ``IP'')
st	5	ST		# ST datagram mode
tcp	6	TCP		# transmission control protocol
egp	8	EGP		# exterior gateway protocol
igp	9	IGP		# any private interior gateway (Cisco)
pup	12	PUP		# PARC universal packet protocol
udp	17	UDP		# user datagram protocol
...

$> man protocols

List of protocols supported in Linux

Docs for protocols. But is needed rather rare. Usually protocol 0 is used (default depending on situation)

IPC. UNIX domain sockets [3]

int
socketpair(int domain, int type, int protocol,
           int socket_vector[2]);

Socket type describes a way of working with the socket. Almost always corresponds to exactly one protocol.

Existing types:

  • SOCK_DGRAM - the socket is used to send/receive packets of limited size, no delivery and order guarantees. Usually it is UDP.
  • SOCK_STREAM - the socket provides a contiguous byte stream, guarantees delivery and order. Usually it is TCP.
  • SOCK_RAW - like dgram, but packets are read/sent in a 'raw' form. The user needs to build TCP and UDP protocols himself, create and fill the headers. A raw socket can be used, for example, to implement ICMP protocol used by 'ping' command.
  • ...

IPC. UNIX domain sockets [4]

int
socketpair(int domain, int type, int protocol,
           int socket_vector[2]);

Domain defines a protocol family, in which a user can choose a type, and probably a concrete protocol.

Existing domains:

  • AF_UNIX/AF_LOCAL - synonyms of one domain - local machine. Sockets of this type can't use any built-in protocols, because don't use the network at all.
  • AF_INET - protocols based on IPv4.
  • AF_INET6 - protocols based on IPv6.
  • AF_PACKET - 'raw' sockets, the user implements protocols himself.
  • ...

IPC. UNIX domain sockets [5]

#define MAX_MSG_SIZE 1024 * 1024

void worker(int sock)
{
	char buffer[MAX_MSG_SIZE];
	ssize_t size;
	while ((size = read(sock, buffer, sizeof(buffer))) > 0)
		printf("Client received %d\n", (int) size);
}

int main(int argc, const char **argv)
{
        int socket_type = strcmp(argv[1], "stream") == 0 ?
                          SOCK_STREAM : SOCK_DGRAM;
	int sockets[2];
	socketpair(AF_UNIX, socket_type, 0, sockets);
	if (fork() == 0) {
		close(sockets[0]);
		worker(sockets[1]);
		return 0;
	}
	close(sockets[1]);
	char buffer[MAX_MSG_SIZE];
	int size;
	while (scanf("%d", &size) > 0) {
		printf("Server sent %d\n", size);
		write(sockets[0], buffer, size);
	}
	return 0;
}

7_ipc/13_socketpair.c

The user from a keyboard chooses a socket type: packet or stream

A channel is created, like a pipe

A second process forks, and reads everything from the socket

Parent sends buffers of the given sizes to the second process

IPC. UNIX domain sockets [6]

7_ipc/13_socketpair.c

$> gcc 13_socketpair.c

$> ./a.out stream
30000
Server sent 30000
Client received 30000
50000
Server sent 50000
Client received 50000

$> ./a.out dgram
50000
Server sent 50000
Client received 50000
100000
Server sent 100000
Client received 100000
1000000
Server sent 1000000
Client received 219264
Client received 219264
Client received 219264
Client received 219264
Client received 122944

Stream socket sends big data without splitting. Too big data is not sent at all

Packet socket sends data of any size, optionally split into packets

IPC. UNIX domain sockets [7]

int
socket(int domain, int type, int protocol);

int
bind(int sockfd, const struct sockaddr *addr,
     socklen_t addrlen);

int
listen(int sockfd, int backlog);

int
connect(int sockfd, const struct sockaddr *addr,
        socklen_t addrlen);

int
accept(int sockfd, struct sockaddr *addr,
       socklen_t *addrlen);

Creation of a socket with specified domain, protocol, type

Bind a socket to a name. By this name clients will be able to connect

Listening for incoming connections

Connect the socket to an existing server. Then bind() and listen() are not needed

If bind() + listen() were done, the server-socket takes clients via accept()

IPC. UNIX domain sockets [8]

Usage

int fd = socket();
connect(fd, remote_addr);
/* Ready to read/write fd. */

Connection to a named socket

Creation of a named socket without connection

int fd = socket();
bind(fd, addr);
/** Ready to read/write fd. */

Creation of a named socket with connection

int fd = socket();
bind(fd, addr);
listen(fd);
while(1) {
        int remote_fd = accept(fd);
        /*
         * Ready to read/write
         * remote_fd.
         */
}

Connect() creates a paired socket on the server side, and this pair can interact just like socketpair()

int fd2 = socket();
bind(fd2, addr2);
/** Ready to read/write fd2. */

read/write

send/recv

sendto/recvfrom

Only packet sockets work without connect(), and destination address should be specified for each packet manually

IPC. UNIX domain sockets [9]

int
bind(int sockfd, const struct sockaddr *addr,
     socklen_t addrlen);
struct sockaddr {
        sa_family_t sa_family;
        char sa_data[14];
};

Socket name is its address

- AF_UNIX, AF_INET, ...

- data of child structures, padding ...

struct sockaddr_in {
        sa_family_t sin_family;
        in_port_t sin_port;
        struct in_addr sin_addr;
};
struct sockaddr_un {
        sa_family_t sun_family;
        char sun_path[108];
};
struct sockaddr_nl {
        sa_family_t nl_family;
        unsigned short nl_pad;
        pid_t nl_pid;
        __u32 nl_groups;
};

Inheritance of struct sockaddr by its duplication

IPC. UNIX domain sockets [10]

struct base {
        int base_field1;
        char *base_field2;
};

struct derivative {
        struct base base;
        int deriv_field3;
        bool deriv_field4;
};

void do_sth(struct base *obj);

int main()
{
        struct derivative der;
        do_sth((struct base *)&der);
}

Inheritance in C by inclusion

struct base {
        int base_field1;
        char *base_field2;
        char padding[32];
};

struct derivative {
        int base_field1;
        char *base_field2;
        int deriv_field3;
        bool deriv_field4;
};

void do_sth(struct base *obj);

int main()
{
        struct derivative der;
        do_sth((struct base *)&der);
}

Inheritance in C by duplication

Parent structure in a child is stored as a member

Can be cast to the parent, because the struct stores the parent in the beginning

All parent attributes are stored in a child struct

Also can be cast. Beginning of the struct is exactly like the parent anyway

IPC. UNIX domain sockets [11]

struct sockaddr_un {
        sa_family_t sun_family;
        char sun_path[108];
};

Should be AF_UNIX, so as functions, using a pointer at struct sockaddr *, could use sockaddr.sa_family to determine the address type. Like dynamic_cast() in C++.

Path to socket file. If bind() is done, and the file does not exist, it is created. If it exists, then it is an error. The file should be deleted manually after usage.

IPC. UNIX domain sockets [12]

void
initiator_f(int sock, struct sockaddr_un *next_addr)
{
	int number;
	while(scanf("%d", &number) > 0) {
		if (sendto(sock, &number, sizeof(number), 0,
			   (struct sockaddr *) next_addr,
			   sizeof(*next_addr)) == -1) {
			printf("error = %s\n", strerror(errno));
			continue;
		}
		printf("Sent %d\n", number);
		int buffer = 0;
		ssize_t size = recv(sock, &buffer, sizeof(buffer), 0);
		if (size == -1)
			printf("error = %s\n", strerror(errno));
		else
			printf("Received %d\n", buffer);
	}
}

7_ipc/14_sock_dgram.c [1]

Process 1 - initiator

Process 2

Process 3

N

N + 1

N + 2

Initiator reads a number from the keyboard and sends it further along the ring

Initiator waits for a response from its counterpart, and prints the response

void
worker_f(int sock, struct sockaddr_un *next_addr)
{
	while(1) {
		int buffer = 0;
		ssize_t size = recv(sock, &buffer, sizeof(buffer), 0);
		if (size == -1) {
			printf("error = %s\n", strerror(errno));
			continue;
		}
		printf("Received %d\n", buffer);
		buffer++;
		if (sendto(sock, &buffer, sizeof(buffer), 0,
			  (struct sockaddr *) next_addr,
			  sizeof(*next_addr)) == -1)
			printf("error = %s\n", strerror(errno));
		else
			printf("Sent %d\n", buffer);
	}
}

7_ipc/14_sock_dgram.c [2]

Other processes read and resend with +1

int
main(int argc, const char **argv)
{
	if (argc < 2) {
		printf("No id\n");
		return -1;
	}
	int myid = atoi(argv[1]);
	int sock = socket(AF_UNIX, SOCK_DGRAM, 0);
	if (sock == -1) {
		printf("error = %s\n", strerror(errno));
		return -1;
	}
	struct sockaddr_un addr;
	addr.sun_family = AF_UNIX;
	sprintf(addr.sun_path, "sock%d", myid);
	if (bind(sock, (struct sockaddr *) &addr, sizeof(addr)) != 0) {
		printf("error = %s\n", strerror(errno));
		return -1;
	}

	struct sockaddr_un next_addr;
	next_addr.sun_family = AF_UNIX;
	sprintf(next_addr.sun_path, "sock%d", (myid + 1) % 3);

	ssize_t size;
	int buffer;
	if (myid == 0)
		initiator_f(sock, &next_addr);
	else
		worker_f(sock, &next_addr);
	return 0;
}

7_ipc/14_sock_dgram.c [3]

Participants create their sockets and do bind() to sock1, sock2 and sock3 addresses

Determine to whom the packets should be forwarded next along the ring

Process chooses its role by id. Initiator is 0

IPC. UNIX domain sockets [13]

7_ipc/14_sock_dgram.c

$> gcc 14_sock_dgram.c

$> ./a.out 0


$> ./a.out 1


$> ./a.out 2
1
Sent 1
Received 1
Sent 2
Received 2
Sent 3
Received 3

IPC. UNIX domain sockets [14]

7_ipc/15_sock_server.c [1]

void *
worker_f(void *arg)
{
	printf("New client created\n");
	int client_sock = (int) arg;
	while(1) {
		int buffer = 0;
		ssize_t size = recv(client_sock, &buffer, sizeof(buffer), 0);
		if (size == -1) {
			printf("error = %s\n", strerror(errno));
			continue;
		}
		if (size == 0) {
			printf("Closed connection\n");
			break;
		}
		printf("Received %d\n", buffer);
		buffer++;
		if (send(client_sock, &buffer, sizeof(buffer), 0) == -1)
			printf("error = %s\n", strerror(errno));
		else
			printf("Sent %d\n", buffer);
	}
	close(client_sock);
	return NULL;
}

The function works in a separate thread, and serves one client

Client receives a number which should be increased

And sends it back

7_ipc/15_sock_server.c [2]

int main(int argc, const char **argv)
{
	int sock = socket(AF_UNIX, SOCK_STREAM, 0);
	struct sockaddr_un addr;
	addr.sun_family = AF_UNIX;
	sprintf(addr.sun_path, "%s", "sock_server");
	if (bind(sock, (struct sockaddr *) &addr, sizeof(addr)) != 0) {
		printf("error = %s\n", strerror(errno));
		return -1;
	}
	if (listen(sock, 128) == -1) {
		printf("error = %s\n", strerror(errno));
		return -1;
	}
	pthread_attr_t attr;
	pthread_attr_init(&attr);
	pthread_attr_setdetachstate(&attr, 1);
	while(1) {
		pthread_t worker_thread;
		int client_sock = accept(sock, NULL, NULL);
		if (client_sock == -1) {
			printf("error = %s\n", strerror(errno));
			continue;
		}
		pthread_create(&worker_thread, &attr, worker_f,
	                       (void *) client_sock);
	}
	pthread_attr_destroy(&attr);
	close(sock);
	return 0;
}

Server does socket() + bind() + listen() to listen for clients

Each client is placed into a separate thread

Use accept() to connect incoming clients

IPC. UNIX domain sockets [15]

7_ipc/15_sock_client.c

int main()
{
	int sock = socket(AF_UNIX, SOCK_STREAM, 0);
	struct sockaddr_un addr;
	addr.sun_family = AF_UNIX;
	sprintf(addr.sun_path, "%s", "sock_server");

	if (connect(sock, (struct sockaddr *) &addr, sizeof(addr)) != 0) {
		printf("error = %s\n", strerror(errno));
		return -1;
	}
	int number;
	while (scanf("%d", &number) > 0) {
		send(sock, &number, sizeof(number), 0);
		printf("Sent %d\n", number);
		number = 0;
		int rc = recv(sock, &number, sizeof(number), 0);
		if (rc == 0) {
			printf("Closed connection\n");
			break;
		}
		if (rc == -1)
			printf("error = %s\n", strerror(errno));
		else
			printf("Received %d\n", number);
	}
	close(sock);
	return 0;
}

Returned numbers are expected to be increased by 1

socket() + connect() to connect to a server

Numbers from a keyboard are sent to the server

IPC. UNIX domain sockets [16]

7_ipc/15_sock_client.c

$> gcc 15_sock_server.c
        -o server
$> ./server
$> gcc 15_sock_client.c
        -o client
$> ./client


$> ./client
New client created
1
Sent 1
Received 2
100
Sent 100
Received 101
Received 100
Sent 101
New client created
Received 1
Sent 2
^C

$>
Closed connection
^C

$>
Closed connection

Summary

IPC can be used for process cooperation, for parallel work. Although now threads are preferable for this.

IPC can be anonymous or named. The named are 3 from XSI, 3 from POSIX. XSI is quite bad except maybe for shared memory. POSIX domain sockets are used widely.

Sockets can be of multiple types. Domain is one of them. Local this system. Not related to networking but do behave like real network sockets.

Conclusion

Lectures: slides.com/gerold103/decks/sysprog_eng

Next time:

Network. Models TCP/IP, OSI. Relation to the kernel. Interfaces and examples


Press on the heart, if like the lecture

System programming 7

By Vladislav Shpilevoy

System programming 7

IPC. Pipe, FIFO. XSI: message queue, semaphore, shared memory. POSIX semaphore. Sockets: API, byte ordering. Domain sockets.

  • 1,475

More from Vladislav Shpilevoy