diff --git a/.dockerignore b/.dockerignore index 6b8710a..e0ba315 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ .git +node_modules \ No newline at end of file diff --git a/.gitignore b/.gitignore index efc5ba0..3656e9c 100644 --- a/.gitignore +++ b/.gitignore @@ -141,4 +141,7 @@ Icon .AppleDesktop Network Trash Folder Temporary Items -.apdisk \ No newline at end of file +.apdisk + +node_modules/ +.localstack/ \ No newline at end of file diff --git a/README.md b/README.md index 68ed4e6..e423c64 100644 --- a/README.md +++ b/README.md @@ -93,3 +93,25 @@ sudo docker tag mlflow-basis:latest /mlflow-basis:/mlflow-basis: ``` + +AWS S3 based [on this article ](https://dev.to/goodidea/how-to-fake-aws-locally-with-localstack-27me) + + +1. [install aws cli](https://aws.amazon.com/cli/) + + +``` +aws configure +AWS Access Key ID [****************123]: AKIAIOSFODNN7EXAMPLE +AWS Secret Access Key [****************123]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +Default region name [us-west-2]: us-east-1 +Default output format [json]: +``` + + +```shell +npm i +aws --endpoint-url=http://localhost:9000 s3 mb s3://mlflow +aws --endpoint-url=http://localhost:9000 s3api put-bucket-acl --bucket mlflow --acl public-read +``` + diff --git a/another/docker-compose.yml b/another/docker-compose.yml new file mode 100644 index 0000000..b9a462d --- /dev/null +++ b/another/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3' +services: + sftp: + image: atmoz/sftp + #volumes: + # - /upload:/home/foo/upload + ports: + - "2222:22" + command: ftpuser:ftppass:::0 + networks: + - A + database: + container_name: pg_mlflow + image: postgres:13 + environment: + - POSTGRES_USER=pgsql + - POSTGRES_PASSWORD=pg_password + - POSTGRES_DB=mlflow + networks: + - A + tracker: + container_name: tracker_mlflow + image: tracker_ml + build: + context: ./mlflow + dockerfile: Dockerfile + ports: + - "5000:5000" + networks: + - A + + entrypoint: mlflow server --default-artifact-root sftp://ftpuser:ftppass@localhost:2222 -h 0.0.0.0 +networks: + A: + driver: bridge \ No newline at end of file diff --git a/test.sh b/another/test.sh similarity index 100% rename from test.sh rename to another/test.sh diff --git a/aws.js b/aws.js new file mode 100644 index 0000000..bca9c49 --- /dev/null +++ b/aws.js @@ -0,0 +1,46 @@ +const AWS = require('aws-sdk') +require('dotenv').config() + +const credentials = { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_KEY, +} + +const useLocal = process.env.NODE_ENV !== 'production' + +const bucketName = process.env.AWS_BUCKET_NAME + +const s3client = new AWS.S3({ + credentials, + /** + * When working locally, we'll use the Localstack endpoints. This is the one for S3. + * A full list of endpoints for each service can be found in the Localstack docs. + */ + endpoint: useLocal ? 'http://localhost:4572' : undefined, + /** + * Including this option gets localstack to more closely match the defaults for + * live S3. If you omit this, you will need to add the bucketName to the `Key` + * property in the upload function below. + * + * see: https://github.com/localstack/localstack/issues/1180 + */ + s3ForcePathStyle: true, +}) + + +const uploadFile = async (data, fileName) => + new Promise((resolve) => { + s3client.upload( + { + Bucket: bucketName, + Key: fileName, + Body: data, + }, + (err, response) => { + if (err) throw err + resolve(response) + }, + ) + }) + +module.exports = uploadFile \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b9a462d..199b0ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,15 @@ -version: '3' +version: '3.2' services: - sftp: - image: atmoz/sftp - #volumes: - # - /upload:/home/foo/upload - ports: - - "2222:22" - command: ftpuser:ftppass:::0 - networks: - - A - database: - container_name: pg_mlflow - image: postgres:13 - environment: - - POSTGRES_USER=pgsql - - POSTGRES_PASSWORD=pg_password - - POSTGRES_DB=mlflow - networks: - - A - tracker: - container_name: tracker_mlflow - image: tracker_ml - build: - context: ./mlflow - dockerfile: Dockerfile - ports: - - "5000:5000" - networks: - - A - - entrypoint: mlflow server --default-artifact-root sftp://ftpuser:ftppass@localhost:2222 -h 0.0.0.0 -networks: - A: - driver: bridge \ No newline at end of file + localstack: + image: minio/minio:latest + container_name: aws-s3 + ports: + - 9000:9000 + environment: + - MINIO_ACCESS_KEY=${AWS_ACCESS_KEY_ID} + - MINIO_SECRET_KEY=${AWS_SECRET_KEY} + command: + server /date + #volumes: + # - './.localstack:/tmp/localstack' + # - '/var/run/docker.sock:/var/run/docker.sock' \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..22fa9c6 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,105 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "aws-sdk": { + "version": "2.738.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.738.0.tgz", + "integrity": "sha512-oO1odRT4DGssivoP6lHO3yB6I+5sU0uqpj6UJ0kS5wrHQ9J9EGrictLVKA9y6XhN0sNW+XPNLD9jMMY/A+gXWA==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } +} diff --git a/test-upload.js b/test-upload.js new file mode 100644 index 0000000..cddd4ad --- /dev/null +++ b/test-upload.js @@ -0,0 +1,19 @@ +const fs = require('fs') +const path = require('path') +const uploadFile = require('./aws') + +const testUpload = () => { + const filePath = path.resolve(__dirname, 'test_upload.webp') + const fileStream = fs.createReadStream(filePath) + const now = new Date() + const fileName = `test-image-${now.toISOString()}.jpg` + uploadFile(fileStream, fileName).then((response) => { + console.log(":)") + console.log(response) + }).catch((err) => { + console.log(":|") + console.log(err) + }) +} + +testUpload() \ No newline at end of file diff --git a/test_upload.webp b/test_upload.webp new file mode 100644 index 0000000..a7c7a84 Binary files /dev/null and b/test_upload.webp differ