.drone.yml

# #########################################################
#                       LINT TEST
# #########################################################
kind: pipeline
name: static-tests

trigger:
  events:
  - push
  - cron
  - tag

steps:
- name: flake8
  image: alpine/flake8:3.9.1
  commands:
  - flake8
- name: pylint
  image: cytopia/pylint:latest-0.6
  commands:
  - pylint snoop

---
# #########################################################
#                       build arm64
# #########################################################

kind: pipeline
name: build arm64

depends_on:
- static-tests

platform:
  os: linux
  arch: arm64

trigger:
  events:
  - push
  - cron
  - tag

concurrency:
  limit: 2

clone:
  depth: 5000

steps:
- name: fetch
  image: alpine/git
  commands:
  - git fetch --tags

- name: build base tag
  image: plugins/docker
  settings:
    repo: liquidinvestigations/hoover-snoop2
    auto_tag: true
    auto_tag_suffix: base-arm64
    dockerfile: Dockerfile.base
    username:
      from_secret: docker_username
    password:
      from_secret: docker_password
  when:
    event:
    - tag
  depends_on:
  - fetch

- name: build base master branch
  image: plugins/docker
  settings:
    repo: liquidinvestigations/hoover-snoop2
    tags: 0-base-arm64
    dockerfile: Dockerfile.base
    username:
      from_secret: docker_username
    password:
      from_secret: docker_password
  when:
    event:
    - push
    - cron
    branch:
    - master
  depends_on:
  - fetch

- name: docker push branch
  image: plugins/docker
  settings:
    repo: liquidinvestigations/hoover-snoop2
    tags: ${DRONE_COMMIT_BRANCH}-arm64
    build_args:
    - BASE_TAG=0-base-arm64
    username:
      from_secret: docker_username
    password:
      from_secret: docker_password
  when:
    event:
    - push
    - cron
  depends_on:
  - build base master branch
  - fetch

- name: docker push autotag + latest
  image: plugins/docker
  settings:
    repo: liquidinvestigations/hoover-snoop2
    auto_tag: true
    auto_tag_suffix: arm64
    build_args:
    - BASE_TAG=0-base-arm64
    username:
      from_secret: docker_username
    password:
      from_secret: docker_password
  depends_on:
  - build base master branch
  - fetch

---
# #########################################################
#                       build x86
# #########################################################

kind: pipeline
name: build x86

depends_on:
- static-tests

trigger:
  events:
  - push
  - cron
  - tag

concurrency:
  limit: 2

clone:
  depth: 5000

steps:
- name: fetch
  image: alpine/git
  commands:
  - git fetch --tags

- name: build base tag
  image: plugins/docker
  settings:
    repo: liquidinvestigations/hoover-snoop2
    auto_tag: true
    auto_tag_suffix: base
    dockerfile: Dockerfile.base
    username:
      from_secret: docker_username
    password:
      from_secret: docker_password
  when:
    event:
    - tag
  depends_on:
  - fetch

- name: build base master branch
  image: plugins/docker
  settings:
    repo: liquidinvestigations/hoover-snoop2
    tags: 0-base
    dockerfile: Dockerfile.base
    username:
      from_secret: docker_username
    password:
      from_secret: docker_password
  when:
    event:
    - push
    - cron
    branch:
    - master
  depends_on:
  - fetch

- name: docker push branch
  image: plugins/docker
  settings:
    repo: liquidinvestigations/hoover-snoop2
    tags: ${DRONE_COMMIT_BRANCH}
    username:
      from_secret: docker_username
    password:
      from_secret: docker_password
  when:
    event:
    - push
    - cron
  depends_on:
  - build base master branch
  - fetch

- name: docker push autotag + latest
  image: plugins/docker
  settings:
    repo: liquidinvestigations/hoover-snoop2
    auto_tag: true
    username:
      from_secret: docker_username
    password:
      from_secret: docker_password
  depends_on:
  - build base master branch
  - fetch

---
# #########################################################
#                   build multiplatform
# #########################################################

kind: pipeline
name: build multiplatform

depends_on:
- build x86
- build arm64

clone:
  disable: true


steps:
- name: docker multiplatform branch
  image: docker
  pull: if-not-exists
  environment:
    IMG: "liquidinvestigations/hoover-snoop2:${DRONE_COMMIT_BRANCH}"
    DOCKER_USERNAME:
      from_secret: docker_username
    DOCKER_PASSWORD:
      from_secret: docker_password
  commands:
  - "docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD"
  - "docker buildx imagetools  create --append $IMG --append $IMG-arm64 --tag  $IMG"
  when:
    event:
    - push
    - cron

- name: docker multiplatform tag
  image: docker
  pull: if-not-exists
  environment:
    IMG: "liquidinvestigations/hoover-snoop2:${DRONE_TAG:1}"
    DOCKER_USERNAME:
      from_secret: docker_username
    DOCKER_PASSWORD:
      from_secret: docker_password
  commands:
  - "docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD"
  - "docker buildx imagetools  create --append $IMG --append $IMG-arm64 --tag  $IMG"
  when:
    event:
    - tag
---

# #########################################################
#                       test
# #########################################################

kind: pipeline
name: test

depends_on:
- build multiplatform

concurrency:
  limit: 1

trigger:
  events:
  - push
  - cron
  - tag

services:
- name: minio-blobs
  image: minio/minio:RELEASE.2022-03-26T06-49-28Z
  volumes:
  - name: minio-v
    path: /data
  commands:
  - minio server /data

- name: minio-collections
  image: minio/minio:RELEASE.2022-03-26T06-49-28Z
  volumes:
  - name: test-collections-v
    path: /data
  commands:
  - minio server /data

- name: broken-filename-service
  image: liquidinvestigations/broken-filename-service:0.0.6
  volumes:
  - name: test-collections-v
    path: /data
  commands:

- name: snoop-rabbitmq
  image: rabbitmq:3.8.5-management-alpine
  volumes:
  - name: rabbit-v
    path: /var/lib/rabbitmq

- name: snoop-nlp-service
  image: liquidinvestigations/nlp-service:main
  pull: always
  environment:
    NLP_SERVICE_PRESET: "full_md"
    NLP_SERVICE_FALLBACK_LANGUAGE: "en"
  volumes:
  - name: nlp-service
    path: /data
  commands:
  - /opt/app/runserver

- name: snoop-tika
  image: logicalspark/docker-tikaserver:1.28.1

- name: snoop-thumbnail
  image: fpurchess/preview-service
  user: root #undocumented setting "user" see: https://discourse.drone.io/t/permission-denied/6732/4
  environment: 
    WEB_CONCURRENCY: 20
  volumes:
  - name: thumbnails-files
    path: /tmp/files/
  - name: thumbnails-cache
    path: /tmp/cache/

- name: snoop-pdf-preview
  image: thecodingmachine/gotenberg:7

- name: snoop-image-classification
  image: liquidinvestigations/image-classification-service:0.0.2
  environment: 
    OBJECT_DETECTION_ENABLED: true
    OBJECT_DETECTION_MODEL: "yolo"
    IMAGE_CLASSIFICATION_ENABLED: true
    IMAGE_CLASSIFICATION_MODEL: "mobilenet"
    WAITRESS_THREADS: 50

- name: search-es
  image: docker.elastic.co/elasticsearch/elasticsearch:6.2.4
  environment:
    discovery.type: single-node
    cluster.routing.allocation.disk.watermark.low: "97%"
    cluster.routing.allocation.disk.watermark.high: "98%"
    cluster.routing.allocation.disk.watermark.flood_stage: "99%"
    cluster.info.update.interval: "10m"
  volumes:
  - name: es-v
    path: /var/lib/elasticsearch/data

- name: snoop-pg
  image: postgres:12.2
  environment:
    POSTGRES_USER: snoop
    POSTGRES_DATABASE: snoop
    POSTGRES_PASSWORD: snoop
  volumes:
  - name: pg-v
    path: /var/lib/postgresql/data

- name: translate-batch
  image: liquidinvestigations/libre-translate:v1.2.7-liquid-ci
  environment:
    LT_CHAR_LIMIT: "157286400"
    LT_DISABLE_WEB_UI: "false"
    OMP_NUM_THREADS: "1"
    OMP_THREAD_LIMIT: "1"
    GUNICORN_NUM_WORKERS: "2"

steps:
- name: download testdata
  image: alpine/git:1.0.7
  volumes:
  - name: test-collections-v
    path: /data
  - name: test-collections-v-2
    path: /data-2
  commands:
  - cd /data
  - git clone https://github.com/hoover/testdata
  - cp -a testdata /data-2

- name: wait for nlp service, translation
  image: alpine/git:1.0.7
  commands:
  - apk add curl
  - apk add curl-dev
  - while ! curl "http://snoop-nlp-service:5000/config"; do sleep 30; done
  - while ! curl "http://translate-batch:5000/languages"; do sleep 30; done

- name: run tests
  image: liquidinvestigations/hoover-snoop2:${DRONE_COMMIT_BRANCH}
  # privileged because needs:
  # cap_add = ["mknod", "sys_admin"]
  # devices = [{host_path = "/dev/fuse", container_path = "/dev/fuse"}]
  # security_opt = ["apparmor=unconfined"]
  privileged: true
  pull: always
  volumes:
  - name: dev-fuse
    path: /dev/fuse

  - name: test-collections-v-2
    path: /opt/hoover/collections

  environment:
    WAIT_HOSTS: search-es:9200, snoop-pg:5432, snoop-tika:9998, snoop-thumbnail:9990, snoop-pdf-preview:3000, snoop-image-classification:5001, snoop-nlp-service:5000, minio-blobs:9000, translate-batch:5000, minio-collections:9000
    WAIT_HOSTS_TIMEOUT: 60
    SNOOP_DB: "postgresql://snoop:snoop@snoop-pg:5432/snoop"
    SNOOP_COLLECTIONS: '[{"name": "testdata", "process": true}]'
    SNOOP_AMQP_URL: "amqp://snoop-rabbitmq"
    SNOOP_RABBITMQ_HTTP_URL: "snoop-rabbitmq:15672/"
    SNOOP_TIKA_URL: "http://snoop-tika:9998"
    SNOOP_THUMBNAIL_URL: "http://snoop-thumbnail:8000/"
    SNOOP_PDF_PREVIEW_URL: "http://snoop-pdf-preview:3000/"
    SNOOP_IMAGE_CLASSIFICATION_URL: "http://snoop-image-classification:5001/classify-image"
    SNOOP_OBJECT_DETECTION_URL: "http://snoop-image-classification:5001/detect-objects"
    SNOOP_ES_URL: "http://search-es:9200"
    SNOOP_NLP_URL: "http://snoop-nlp-service:5000"
    SNOOP_EXTRACT_ENTITIES:  True
    SNOOP_DETECT_LANGUAGES: True
    SNOOP_TRANSLATION_URL: "http://translate-batch:5000"
    SNOOP_BROKEN_FILENAME_SERVICE: "http://broken-filename-service:5000"
    OMP_THREAD_LIMIT: 4
    OMP_NUM_THREADS: 4
    SNOOP_OCR_PROCESSES_PER_DOC: 4

    USER_MODE: "TEST"
  commands:
  - whoami
  - cd /opt/hoover/snoop
  - mkdir -p /opt/hoover/snoop/blobs
  - mkdir -p /opt/hoover/collections
  - chown -R 666:666 /opt/hoover/collections
  - py.test -vv

volumes:
- name: dev-fuse
  host:
    path: /dev/fuse
- name: minio-v
  temp: {}
- name: test-collections-v
  temp: {}
- name: test-collections-v-2
  temp: {}
- name: es-v
  temp: {}
- name: pg-v
  temp: {}
- name: thumbnails-files
  temp: {}
- name: thumbnails-cache
  temp: {}
- name: nlp-service
  temp: {}

---
# #########################################################
#                       secrets
# #########################################################

kind: secret
name: docker_username
get:
  path: liquid/ci/drone.docker
  name: username

---
kind: secret
name: docker_password
get:
  path: liquid/ci/drone.docker
  name: password

---
kind: secret
name: ghcr_docker_username
get:
  path: liquid/ci/gh-cr.docker
  name: username

---
kind: secret
name: ghcr_docker_password
get:
  path: liquid/ci/gh-cr.docker
  name: password