the pipeline

about me:

Konstantin Schikarski

platform engineer @ turbinekreuzberg

background in systemadministration

 

 

in love with pipelines

the pipeline

magento2-pipeline turbine style

 

the pipeline

the pipeline

the pipeline

the pipeline

the pipeline

environments deployed to kubernetes

magento2 - pipeline /build

.gitlab-ci.yml

build:
  stage: build
  image: ${DEV_TOOLS_IMAGE}
  services:
    - name: ${DATABASE_IMAGE}
  variables:
    COMPOSER_HOME: $CI_PROJECT_DIR/docker/composer
    DATABASE_HOST: 127.0.0.1
  script:
    - timeout 300 sh -c "while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT} -v; do sleep 0.5; done"
    - mysql -h ${DATABASE_HOST} -u ${MYSQL_ROOT_USER} --password=${MYSQL_ROOT_PASSWORD} < docker/mysql/master_database.sql
    - cd htdocs
    - composer global require "hirak/prestissimo:^0.3"
    - composer install
    - bin/magento setup:config:set -q --db-host=${DATABASE_HOST}
#    - bin/magento deploy:mode:set production
    - bin/magento setup:upgrade
    - bin/magento sampledata:deploy
    - bin/magento setup:static-content:deploy -f de_DE en_US
    - rm -rf phpserver update var/cache/* var/view_preprocessed/* generated/* pub/media/catalog/product/cache/*

  cache:
    paths:
      - docker/composer
  artifacts:
    expire_in: 1 day
    paths:
      - htdocs

magento2 - pipeline /build

magento2 - pipeline

/deploy

k8s

magento2.yml

---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: magento2
  namespace: ${NAMESPACE}
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: magento2
    spec:
      containers:
        - name: magento2
          image: ${APPLICATION_IMAGE}
          env:
            - name: DATABASE_HOST
              value: ${DATABASE_HOST}
            - name: BASE_URL
              value: ${BASE_URL}
          ports:
            - containerPort: 80
              name: http
          livenessProbe:
            tcpSocket:
              port: 80
            initialDelaySeconds: 120
          readinessProbe:
            tcpSocket:
              port: 80
            initialDelaySeconds: 30

pipeline - building blocks

pipeline - building blocks

gitlab-runner is the worker

pipeline - building blocks

kubernetes is the workhorse

pipeline - building blocks

registry stores the images

pipeline - building blocks

container-image-builder

guess what ...

pipeline - building blocks

... builds images

 let’s build it

deploy the blocks

 

start pipeline in git repo

 

kubernetes @ syseleven

gitlabrunner

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: runner
  namespace: gitlab-runner
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: runner
    spec:
      serviceAccountName: gitlab-runner
      containers:
        - name: runner
          image: gitlab/gitlab-runner:v10.7.1
          args:
            - "run"
          env:
            - name: METRICS_SERVER
              value: :80
          volumeMounts:
            - name: config
              mountPath: /etc/gitlab-runner
          ports:
            - containerPort: 80
              name: prometheus
          livenessProbe:
            exec:
              command:
                - /bin/sh
                - -c
                - "/entrypoint verify 2>&1 | grep 'is alive'"
      volumes:
        - name: config
          configMap:
apiVersion: v1
kind: ConfigMap
metadata:
  name: runner
  namespace: gitlab-runner
data:
# created via docker run -ti -v `pwd`/config:/etc/gitlab-runner gitlab/gitlab-runner:alpine-v9.0.0 register
  config.toml: |
    concurrent = 24
    check_interval = 0

    [[runners]]
      name = "konstis-runner"
      url = "https://git.votum-media.net/"
      token = "e06a0001ab76661667x667x667x6671234567890"
      executor = "kubernetes"
      environment = ["DOCKER_HOST=tcp://docker.container-image-builder.svc:2375", "DOCKER_REGISTRY_INTERNAL=registry.registry.svc:5000", "DOCKER_IMAGE=docker:17.12.1-ce"]
      pre_build_script = """
      mkdir -p $HOME/.docker
      echo "{\
        \\"HttpHeaders\\": {\
          \\"GitBranchName\\": \\"$CI_COMMIT_REF_SLUG\\"\
        }\
      }" > $HOME/.docker/config.json
      """
      [runners.cache]
      [runners.kubernetes]
        host = ""
        image = ""
        namespace = "gitlab-runner"
        privileged = false
        service_account_overwrite_allowed = ""
        cpu_limit = "1"
        memory_limit = "2Gi"
        cpu_request = "500m"
        memory_request = "500Mi"
        service_cpu_limit = "1"
        service_memory_limit = "1Gi"
        service_cpu_request = "500m"
        service_memory_request = "500Mi"
        helper_cpu_limit = "1"
        helper_memory_limit = "1Gi"
        helper_cpu_request = "600m"
        helper_memory_request = "500Mi"
        poll_timeout = 800
        [runners.kubernetes.volumes]

config.yaml

docker registry

---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: registry
  namespace: registry
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        name: registry
        readonly: "false"
    spec:
      containers:
      - name: registry
        image: registry:2.6.2
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
        env:
        - name: REGISTRY_HTTP_ADDR
          value: :5000
        - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
          value: /var/lib/registry
        - name: REGISTRY_STORAGE_CACHE_BLOBDESCRIPTOR
          value: redis
        - name: REGISTRY_REDIS_ADDR
          value: redis:6379
        - name: REGISTRY_STORAGE_DELETE_ENABLED
          value: "true"
        volumeMounts:
        - name: image-store
          mountPath: /var/lib/registry
        ports:
        - containerPort: 5000
          name: registry
          protocol: TCP
      volumes:
      - name: image-store
        cinder:
        persistentVolumeClaim:
          claimName: registry-pvc

registry.yaml

storage-class.yml

---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: generic
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
parameters:
  availability: dbl1
  type: quobyte
provisioner: kubernetes.io/cinder

container-image-builder

#dispatcher.yaml
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: dispatcher
  namespace: container-image-builder
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: dispatcher
    spec:
      containers:
        - name: dispatcher
          image: utopiaplanitia/docker-image-builder-dispatcher:1.4.0
          args:
            - --workers=http://builder-0.builder.container-image-builder.svc.cluster.local:2375,http://builder-1.builder.container-image-builder.svc.cluster.local:2375,http://builder-2.builder.container-image-builder.svc.cluster.local:2375
            - --network=host
            - --memory=4294967296
            - --cpu=400000
          ports:
            - containerPort: 2375
              name: docker
          livenessProbe:
            tcpSocket:
              port: 2375
          readinessProbe:
            httpGet:
              path: /_ping
              port: 2375

dispatcher.yaml

review deploy magento2

branches deployed

The Whole Story about Orchestration and CI with Openstack, Kubernetes and Gitlab

 

1. Kubernetes (Steffen@SysEleven)
 

2. CI Pipeline (Konstantin@Turbine)
 

3. Container Builder (David)

David and the

image-builder

the pipeline

By konstantinschikarski