FROM pytorch/pytorch:latest
RUN git clone https://github.com/NVIDIA/apex
RUN cd apex && \
    python3 setup.py install && \
    pip install -v --no-cache-dir --global-option="--cpp_ext" \
    --global-option="--cuda_ext" ./

WORKDIR /fancy-nlp-project
RUN git clone https://github.com/huggingface/transformers.git && \
    cd transformers && \
    python3 -m pip install --no-cache-dir

---

#!/bin/bash
#SBATCH -J JobName
#SBATCH --time=11:00:00      # Czas, kiedy należy uruchomić zadanie
#SBATCH --mem-per-cpu=4096   # Pamięć (w MB) przydzielona do każdego CPU
#SBATCH --cpus-per-task=4    # Liczba rdzeni

---

from datetime import datetime, timedelta

from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.providers.docker.operators.docker import DockerOperator

dag = DAG(
    'docker_sample',
    default_args={'retries': 1},
    schedule_interval=timedelta(minutes=10),
    start_date=datetime(2021, 1, 1),
    catchup=False,
)

t1 = BashOperator(task_id='print_date', bash_command='date', dag=dag)
t2 = BashOperator(task_id='sleep', bash_command='sleep 5', retries=3, dag=dag)
t3 = DockerOperator(
    docker_url='tcp://localhost:2375',  # Adres URL Twojego dockera
    command='/bin/sleep 30',
    image='centos:latest',
    network_mode='bridge',
    task_id='docker_op_tester',
    dag=dag,
)

t4 = BashOperator(
    task_id='print_hello', 
    bash_command='echo "witaj, świecie!!!"', 
    dag=dag
)

t1 >> t2
t1 >> t3
t3 >> t4

---

apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: coinflip-
  annotations:
    workflows.argoproj.io/description: |
      Jest to przykład procesu rzutu monetą, zdefiniowanego jako sekwencja kroków warunkowych. Możesz go również uruchomić w Pythonie:
      https://couler-proj.github.io/couler/examples/#coin-flip
spec:
  entrypoint: coinflip
  templates:
  - name: coinflip
    steps:
    - - name: flip-coin
        template: flip-coin
    - - name: heads
        template: heads
        when: "{{steps.flip-coin.outputs.result}} == heads"
      - name: tails
        template: tails
        when: "{{steps.flip-coin.outputs.result}} == tails"

    - name: flip-coin
      script:
        image: python:alpine3.6
        command: [python]
        source: |
          import random
          result = "heads" if random.randint(0,1) == 0 else "tails"
          print(result)
    - name: heads
      container:
        image: alpine:3.6
        command: [sh, -c]
        args: ["echo \"wypadł orzeł\""]

    - name: tails
      container:
        image: alpine:3.6
        command: [sh, -c]
        args: ["echo \"wypadła reszka\""]
---

# Przykład systemu rekomendującego, który wykorzystuje zespół złożony z dwóch modeli. 
# Model A zostanie uruchomiony na lokalnej maszynie, a model B w chmurze AWS.

class RecSysFlow(FlowSpec):
    @step
    def start(self):
        self.data = load_data()
        self.next(self.fitA, self.fitB)
    # fitA wymaga innej wersji biblioteki NumPy niż fitB
    @conda(libraries={"scikit-learn":"0.21.1", "numpy":"1.13.0"})
    @step
    def fitA(self):
        self.model = fit(self.data, model="A")
        self.next(self.ensemble)

    @conda(libraries={"numpy":"0.9.8"})
    # Wymaga 2 GPU z 16 GB pamięci
    @batch(gpu=2, memory=16000)
    @step
    def fitB(self):
        self.model = fit(self.data, model="B")
        self.next(self.ensemble)

    @step
    def ensemble(self, inputs):
        self.outputs = (
                   (inputs.fitA.model.predict(self.data) +   
                    inputs.fitB.model.predict(self.data)) / 2
                   for input in inputs
        )
        self.next(self.end)

    def end(self):
        print(self.outputs)
