Update docker-compose.yml

fix build problem in macos m1

.github/workflows/verify-docker-compose.yml

@@ -4,7 +4,7 @@ jobs:
   verify:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Show the config
         run: docker-compose config
       - name: Run

README.md

@@ -9,7 +9,7 @@
 3. Open up http://localhost:5000 for MlFlow, and http://localhost:9001/ to browse your files in S3 artifact store
 
 
-**👇Video tutorial how to set it up on Microsoft Azure 👇**
+**👇Video tutorial how to set it up + BONUS with Microsoft Azure 👇**
 
 [![Youtube tutorial](https://user-images.githubusercontent.com/9840635/144674240-f1ede224-410a-4b77-a7b8-450f45cc79ba.png)](https://www.youtube.com/watch?v=ma5lA19IJRA)
 
@@ -28,7 +28,7 @@
 
 1. Configure your client-side
 
-For running mlflow files you need various environment variables set on the client side. To generate them user the convienience script `./bashrc_install.sh`, which installs it on your system or `./bashrc_generate.sh`, which just displays the config to copy & paste.
+For running mlflow files you need various environment variables set on the client side. To generate them use the convienience script `./bashrc_install.sh`, which installs it on your system or `./bashrc_generate.sh`, which just displays the config to copy & paste.
 
 > $ ./bashrc_install.sh   
 > [ OK ] Successfully installed environment variables into your .bashrc!

create_bucket.py

@@ -9,20 +9,20 @@ minioUrl =  os.environ.get('MLFLOW_S3_ENDPOINT_URL')
 bucketName =  os.environ.get('AWS_BUCKET_NAME')
 
 if accessID == None:
-    print('[!] AWS_ACCESS_KEY_ID environemnt variable is empty! run \'source .env\' to load it from the .env file')
+    print('[!] AWS_ACCESS_KEY_ID environment variable is empty! run \'source .env\' to load it from the .env file')
     exit(1)
 
 if accessSecret == None:
-    print('[!] AWS_SECRET_ACCESS_KEY environemnt variable is empty! run \'source .env\' to load it from the .env file')
+    print('[!] AWS_SECRET_ACCESS_KEY environment variable is empty! run \'source .env\' to load it from the .env file')
     exit(1)
 
 if minioUrl == None:
-    print('[!] MLFLOW_S3_ENDPOINT_URL environemnt variable is empty! run \'source .env\' to load it from the .env file')
+    print('[!] MLFLOW_S3_ENDPOINT_URL environment variable is empty! run \'source .env\' to load it from the .env file')
     exit(1)
 
     
 if bucketName == None:
-    print('[!] AWS_BUCKET_NAME environemnt variable is empty! run \'source .env\' to load it from the .env file')
+    print('[!] AWS_BUCKET_NAME environment variable is empty! run \'source .env\' to load it from the .env file')
     exit(1)
 
 minioUrlHostWithPort = minioUrl.split('//')[1]

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   s3:
-    image:  minio/minio:RELEASE.2021-11-24T23-19-33Z
+    image: minio/minio:RELEASE.2023-11-01T18-37-25Z
     restart: unless-stopped
     ports:
       - "9000:9000"
@@ -14,9 +14,9 @@ services:
       - internal
       - public
     volumes:
-      - minio_volume:/data
+      - minio_new_volume:/data
   db:
-    image: mysql/mysql-server:5.7.28
+    image: mysql:8-oracle # -oracle tag supports arm64 architecture!
     restart: unless-stopped
     container_name: mlflow_db
     expose:
@@ -27,16 +27,13 @@ services:
       - MYSQL_PASSWORD=${MYSQL_PASSWORD}
       - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}
     volumes:
-      - db_volume:/var/lib/mysql
+      - db_new_volume:/var/lib/mysql
     networks:
       - internal
   mlflow:
+    image: ubuntu/mlflow:2.1.1_1.0-22.04
     container_name: tracker_mlflow
-    image: tracker_ml
     restart: unless-stopped
-    build:
-      context: ./mlflow
-      dockerfile: Dockerfile
     ports:
       - "5000:5000"
     environment:
@@ -47,7 +44,10 @@ services:
     networks:
       - public
       - internal
-    entrypoint: bash ./wait-for-it.sh db:3306 -t 90 -- mlflow server --backend-store-uri mysql+pymysql://${MYSQL_USER}:${MYSQL_PASSWORD}@db:3306/${MYSQL_DATABASE} --default-artifact-root s3://${AWS_BUCKET_NAME}/ -h 0.0.0.0
+    entrypoint: mlflow server --backend-store-uri mysql+pymysql://${MYSQL_USER}:${MYSQL_PASSWORD}@db:3306/${MYSQL_DATABASE} --default-artifact-root s3://${AWS_BUCKET_NAME}/ --artifacts-destination s3://${AWS_BUCKET_NAME}/ -h 0.0.0.0
+    depends_on:
+      wait-for-db:
+        condition: service_completed_successfully
   create_s3_buckets:
     image: minio/mc
     depends_on:
@@ -55,7 +55,34 @@ services:
     entrypoint: >
       /bin/sh -c "
       until (/usr/bin/mc alias set minio http://s3:9000 '${AWS_ACCESS_KEY_ID}' '${AWS_SECRET_ACCESS_KEY}') do echo '...waiting...' && sleep 1; done;
-      /usr/bin/mc mb minio/mlflow;
+      /usr/bin/mc mb minio/${AWS_BUCKET_NAME};
+      exit 0;
+      "
+    networks:
+      - internal
+  wait-for-db:
+    image: atkrad/wait4x
+    depends_on:
+      - db
+    command: tcp db:3306 -t 90s -i 250ms
+    networks:
+      - internal
+  run_test_experiment:
+    build:
+      context: ./test_experiment
+      dockerfile: Dockerfile
+    platform: linux/amd64  # once continuumio/miniconda3:latest image work on native aarch64 (arm), remove this line 
+    depends_on:
+      - "mlflow"
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_DEFAULT_REGION=${AWS_REGION}
+      - MLFLOW_S3_ENDPOINT_URL=http://s3:9000
+      - MLFLOW_TRACKING_URI=http://mlflow:5000
+    entrypoint: >
+      /bin/sh -c "
+      python3 mlflow_tracking.py;
       exit 0;
       "
     networks:
@@ -65,5 +92,5 @@ networks:
   public:
     driver: bridge
 volumes:
-  db_volume:
+  db_new_volume:
-  minio_volume:
+  minio_new_volume:

mlflow/Dockerfile

@@ -1,10 +0,0 @@
-FROM continuumio/miniconda3:latest
-
-RUN pip install mlflow boto3 pymysql
-
-ADD . /app
-WORKDIR /app
-
-COPY wait-for-it.sh wait-for-it.sh 
-RUN chmod +x wait-for-it.sh
-

mlflow/wait-for-it.sh

@@ -1,182 +0,0 @@
-#!/usr/bin/env bash
-# Use this script to test if a given TCP host/port are available
-
-WAITFORIT_cmdname=${0##*/}
-
-echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
-
-usage()
-{
-    cat << USAGE >&2
-Usage:
-    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-    -h HOST | --host=HOST       Host or IP under test
-    -p PORT | --port=PORT       TCP port under test
-                                Alternatively, you specify the host and port as host:port
-    -s | --strict               Only execute subcommand if the test succeeds
-    -q | --quiet                Don't output any status messages
-    -t TIMEOUT | --timeout=TIMEOUT
-                                Timeout in seconds, zero for no timeout
-    -- COMMAND ARGS             Execute command with args after the test finishes
-USAGE
-    exit 1
-}
-
-wait_for()
-{
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    else
-        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
-    fi
-    WAITFORIT_start_ts=$(date +%s)
-    while :
-    do
-        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
-            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
-            WAITFORIT_result=$?
-        else
-            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
-            WAITFORIT_result=$?
-        fi
-        if [[ $WAITFORIT_result -eq 0 ]]; then
-            WAITFORIT_end_ts=$(date +%s)
-            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
-            break
-        fi
-        sleep 1
-    done
-    return $WAITFORIT_result
-}
-
-wait_for_wrapper()
-{
-    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
-    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    else
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    fi
-    WAITFORIT_PID=$!
-    trap "kill -INT -$WAITFORIT_PID" INT
-    wait $WAITFORIT_PID
-    WAITFORIT_RESULT=$?
-    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    fi
-    return $WAITFORIT_RESULT
-}
-
-# process arguments
-while [[ $# -gt 0 ]]
-do
-    case "$1" in
-        *:* )
-        WAITFORIT_hostport=(${1//:/ })
-        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
-        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
-        shift 1
-        ;;
-        --child)
-        WAITFORIT_CHILD=1
-        shift 1
-        ;;
-        -q | --quiet)
-        WAITFORIT_QUIET=1
-        shift 1
-        ;;
-        -s | --strict)
-        WAITFORIT_STRICT=1
-        shift 1
-        ;;
-        -h)
-        WAITFORIT_HOST="$2"
-        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
-        shift 2
-        ;;
-        --host=*)
-        WAITFORIT_HOST="${1#*=}"
-        shift 1
-        ;;
-        -p)
-        WAITFORIT_PORT="$2"
-        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --port=*)
-        WAITFORIT_PORT="${1#*=}"
-        shift 1
-        ;;
-        -t)
-        WAITFORIT_TIMEOUT="$2"
-        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --timeout=*)
-        WAITFORIT_TIMEOUT="${1#*=}"
-        shift 1
-        ;;
-        --)
-        shift
-        WAITFORIT_CLI=("$@")
-        break
-        ;;
-        --help)
-        usage
-        ;;
-        *)
-        echoerr "Unknown argument: $1"
-        usage
-        ;;
-    esac
-done
-
-if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
-    echoerr "Error: you need to provide a host and port to test."
-    usage
-fi
-
-WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
-WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
-WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
-WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
-
-# Check to see if timeout is from busybox?
-WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
-WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
-
-WAITFORIT_BUSYTIMEFLAG=""
-if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
-    WAITFORIT_ISBUSY=1
-    # Check if busybox timeout uses -t flag
-    # (recent Alpine versions don't support -t anymore)
-    if timeout &>/dev/stdout | grep -q -e '-t '; then
-        WAITFORIT_BUSYTIMEFLAG="-t"
-    fi
-else
-    WAITFORIT_ISBUSY=0
-fi
-
-if [[ $WAITFORIT_CHILD -gt 0 ]]; then
-    wait_for
-    WAITFORIT_RESULT=$?
-    exit $WAITFORIT_RESULT
-else
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        wait_for_wrapper
-        WAITFORIT_RESULT=$?
-    else
-        wait_for
-        WAITFORIT_RESULT=$?
-    fi
-fi
-
-if [[ $WAITFORIT_CLI != "" ]]; then
-    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
-        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
-        exit $WAITFORIT_RESULT
-    fi
-    exec "${WAITFORIT_CLI[@]}"
-else
-    exit $WAITFORIT_RESULT
-fi

quickstart/mlflow_tracking.py

@@ -1,22 +1,22 @@
 import os
 from random import random, randint
 
-from mlflow import mlflow,log_metric, log_param, log_artifacts
+import mlflow
 
 if __name__ == "__main__":
     with mlflow.start_run() as run:
         mlflow.set_tracking_uri('http://localhost:5000')
         print("Running mlflow_tracking.py")
 
-        log_param("param1", randint(0, 100))
+        mlflow.log_param("param1", randint(0, 100))
         
-        log_metric("foo", random())
+        mlflow.log_metric("foo", random())
-        log_metric("foo", random() + 1)
+        mlflow.log_metric("foo", random() + 1)
-        log_metric("foo", random() + 2)
+        mlflow.log_metric("foo", random() + 2)
 
         if not os.path.exists("outputs"):
             os.makedirs("outputs")
         with open("outputs/test.txt", "w") as f:
             f.write("hello world!")
 
-        log_artifacts("outputs")
+        mlflow.log_artifacts("outputs")

test_experiment/Dockerfile

@@ -0,0 +1,6 @@
+FROM continuumio/miniconda3:latest
+
+RUN pip install mlflow boto3
+
+WORKDIR /app
+COPY . .

test_experiment/mlflow_tracking.py

@@ -0,0 +1,22 @@
+import os
+from random import random, randint
+
+import mlflow
+
+if __name__ == "__main__":
+    with mlflow.start_run() as run:
+        mlflow.set_tracking_uri('http://mlflow:5000')
+        print("Running mlflow_tracking.py")
+
+        mlflow.log_param("param1", randint(0, 100))
+        
+        mlflow.log_metric("foo", random())
+        mlflow.log_metric("foo", random() + 1)
+        mlflow.log_metric("foo", random() + 2)
+
+        if not os.path.exists("outputs"):
+            os.makedirs("outputs")
+        with open("outputs/test.txt", "w") as f:
+            f.write("hello world!")
+
+        mlflow.log_artifacts("outputs")